From 0b3bca255797f628af08c13ddc2a2e617759f05a Mon Sep 17 00:00:00 2001 From: Antoine Bernardeau Date: Thu, 5 Nov 2026 09:49:23 +0100 Subject: [PATCH 01/54] vk-registry/dav: share the files object layer Move the object model of the plain-file area (directories, objects, foreign tags, the namespace lock) and the streaming write out of the WebDAV handlers into vk-registry/src/files.rs, so another wire protocol can serve the same objects with its own status codes and bodies. The write is split into staging (stream to uploads/ while hashing) and a commit that promotes and tags under the namespace lock, and the reason a path cannot take an object is returned as a value each front maps to its own status. No behaviour change. --- vk-registry/src/dav/files.rs | 411 +++------------------------------ vk-registry/src/dav/mod.rs | 2 +- vk-registry/src/files.rs | 436 +++++++++++++++++++++++++++++++++++ vk-registry/src/lib.rs | 1 + 4 files changed, 475 insertions(+), 375 deletions(-) create mode 100644 vk-registry/src/files.rs diff --git a/vk-registry/src/dav/files.rs b/vk-registry/src/dav/files.rs index 8931b3e7..021ccc74 100644 --- a/vk-registry/src/dav/files.rs +++ b/vk-registry/src/dav/files.rs @@ -1,165 +1,30 @@ -//! Plain files under `/dav/files//` share the OCI pool. Directories map to repositories -//! under `files/`; each leaf is a tag on a single-layer raw-file manifest, as written by -//! `/upload`, whose layer holds the file bytes. `PUT` streams to `uploads/` while hashing, -//! promotes the blob, then writes the manifest and tag. `MKCOL` creates an empty repository; -//! `DELETE` removes a tag or an empty repository. Authorization uses the resource's repository, -//! so `files//*` covers a whole tree. Entries expire under gc's tag retention and blob -//! grace windows. +//! The WebDAV view of the plain-file area, under `/dav/files//`, over the object layer in +//! [`crate::files`]: this module maps WebDAV verbs, statuses and bodies onto it. `PUT` stages +//! and commits an object; `MKCOL` creates an empty repository; `DELETE` removes a tag or an +//! empty repository. Authorization uses the resource's repository, so `files//*` covers +//! a whole tree. -use std::path::PathBuf; -use std::sync::{Arc, Mutex, PoisonError}; -use std::time::{Duration, SystemTime}; +use std::sync::Arc; +use std::time::Duration; use anyhow::{Context, Result}; -use bytes::Bytes; -use http_body_util::BodyExt; use hyper::body::Incoming; use hyper::header::{CONTENT_DISPOSITION, CONTENT_LENGTH, LAST_MODIFIED}; use hyper::{Request, Response, StatusCode}; -use sha2::{Digest, Sha256}; -use tokio::io::{AsyncWrite, AsyncWriteExt}; use super::{ ALLOWED_FILES, ALLOWED_READ, Depth, Entry, blocking, created, href, http_date, method_not_allowed, multistatus, no_content, not_found, options, propfind_depth, refuse, }; -use crate::{Authz, Body, RepoRemoval, STREAM_CHUNK, ServerState, Store, accounts, error_response}; - -/// The repository every `/dav/files/` path lives under. -pub(super) const FILES_REPO: &str = "files"; - -/// Maximum object size, checked both against Content-Length and while streaming. -const MAX_OBJECT: u64 = 4 << 30; +use crate::files::{ + FILES_REPO, MAX_OBJECT, Node, Object, PutError, Unwritable, Written, allowed, drain_into, + namespace, object_of, repo_name, resolve, under_object, +}; +use crate::{Authz, Body, RepoRemoval, ServerState, Store, accounts, error_response}; /// What every object is served as, matching `/v2/` blobs. const OBJECT_TYPE: &str = "application/octet-stream"; -/// Minimum idle age before a `GET` refreshes the tag's mtime — the "last used" record the -/// gc's retention keys on — so a cache read is a metadata write at most once an hour. -const TOUCH_AFTER: Duration = Duration::from_secs(3600); - -/// Serialize `files/` checks and namespace changes (`PUT` tags, `MKCOL`, `DELETE`) in this -/// server so a `PUT` and `MKCOL` of the same path cannot both succeed. Acquire before the -/// store lock, never while holding it. `/v2/` writes into `files/` do not take this lock. -static NAMESPACE: Mutex<()> = Mutex::new(()); - -/// The namespace held, in this process and against any other serving the store: a registry -/// and the release it hands over to serve it together for a while. -struct Namespace { - _process: std::sync::MutexGuard<'static, ()>, - _store: crate::LockGuard, -} - -/// Hold [`NAMESPACE`], then the store's namespace lock. Accept a poisoned mutex: it guards -/// no data that a panic could leave inconsistent. -fn namespace(store: &Store) -> Result { - let process = NAMESPACE.lock().unwrap_or_else(PoisonError::into_inner); - Ok(Namespace { - _process: process, - _store: store.lock_namespace()?, - }) -} - -/// What a `files/` path names once the store has been looked at. -enum Node { - /// A repository under `files/`, or a path component above one: a directory. - Dir(String), - /// A tag on a raw-file manifest: an object. - Object(Object), - /// A tag on any other manifest — an image pushed over `/v2/` into a `files/` - /// repository. Absent to reads; a write there is 409, so nothing here replaces or - /// deletes it. - Foreign(String), -} - -/// An object's tag and what its manifest says about the layer. -struct Object { - repo: String, - tag: String, - /// the layer blob's hex - layer: String, - /// the layer's canonical length - size: u64, - /// the tag's mtime - modified: SystemTime, -} - -impl Node { - /// The repository this node authorizes as. - fn repo(&self) -> &str { - match self { - Node::Dir(r) | Node::Foreign(r) => r, - Node::Object(o) => &o.repo, - } - } -} - -/// The repository a run of path components names: `files/`. -fn repo_name(segs: &[String]) -> String { - let mut name = String::from(FILES_REPO); - for s in segs { - name.push('/'); - name.push_str(s); - } - name -} - -/// The repository and tag an object at `segs` would have: the last component is the tag. -fn object_of(segs: &[String]) -> Option<(String, &str)> { - let (leaf, parents) = segs.split_last()?; - (!parents.is_empty()).then(|| (repo_name(parents), leaf.as_str())) -} - -/// What is at `segs`: a directory, an object, a foreign tag, or nothing. A directory is any -/// path under `repos/files/` that is one — a repository, or the parent a nested repository's -/// name created — so the tree reads as the client wrote it. -fn resolve(store: &Store, segs: &[String]) -> Result> { - let dir = repo_name(segs); - if store.repo_dir_exists(&dir) { - return Ok(Some(Node::Dir(dir))); - } - let Some((repo, tag)) = object_of(segs) else { - return Ok(None); - }; - let Some((manifest_hex, modified)) = store.tag_target(&repo, tag) else { - return Ok(None); - }; - let layer = match store.get_blob(&manifest_hex)? { - Some(manifest) => crate::raw_file_layer(&manifest), - None => None, - }; - Ok(Some(match layer { - Some((layer, size)) => Node::Object(Object { - repo, - tag: tag.to_string(), - layer, - size, - modified, - }), - None => Node::Foreign(repo), - })) -} - -/// Whether an ancestor of `segs` is a tag. Writing below one would make its path a directory -/// too, and [`resolve`] would then find the directory and hide the tag: RFC 4918 wants a 409 -/// for a parent that is not a collection. A top-level name is always a directory. -fn under_object(store: &Store, segs: &[String]) -> Result { - for n in 2..segs.len() { - if matches!( - resolve(store, &segs[..n])?, - Some(Node::Object(_) | Node::Foreign(_)) - ) { - return Ok(true); - } - } - Ok(false) -} - -/// Whether this caller may `action` on `repo`. -fn allowed(authz: &Authz<'_>, action: accounts::Action, repo: &str) -> bool { - crate::authorize_or_forbidden(authz, action, repo).is_none() -} - /// Serve one `files/` request; `segs` are the decoded components after `/dav/files/`, and /// `collection` whether the path ended in a slash — which an object's path never does. pub(super) async fn route( @@ -339,12 +204,8 @@ fn get( if resp.status() == StatusCode::OK { // A GET served is a use, which is what keeps the tag from the gc's retention. // Refreshed only once idle, so a hot entry is not a metadata write per hit. - if !head - && SystemTime::now() - .duration_since(obj.modified) - .is_ok_and(|idle| idle > TOUCH_AFTER) - { - store.touch_tag(&obj.repo, &obj.tag); + if !head { + crate::files::touch_if_idle(store, &obj); } let headers = resp.headers_mut(); // Prevent uploaded content from rendering on the `/browse` origin. @@ -419,46 +280,15 @@ fn propfind( } } -/// Why a streamed `PUT` stopped short. -enum PutError { - /// the body crossed [`MAX_OBJECT`] — a 413, decided while reading - TooLarge, - /// the read or the write failed — a 500 - Failed(anyhow::Error), -} - -/// A staging file under `uploads/`, unlinked when dropped unless its bytes were promoted — -/// including when the handler's future is dropped with the connection mid-body. -struct Staging { - path: PathBuf, - armed: bool, -} - -impl Drop for Staging { - fn drop(&mut self) { - if self.armed { - // Gone already (a promotion that renamed it, or a discard) is the goal; any - // other failure leaves it to the gc's sweep of idle uploads. - let _ = std::fs::remove_file(&self.path); - } +/// The answer to a `PUT` at a path that takes no object: a top-level name or a directory is +/// 405, a foreign tag there or any tag above it 409. +fn put_refusal(why: Unwritable, href: &str) -> Response { + match why { + Unwritable::TopLevel | Unwritable::Directory => method_not_allowed(href, ALLOWED_FILES), + Unwritable::Foreign | Unwritable::UnderObject => parent_is_object(href), } } -/// Why `segs` cannot take an object, re-checked under [`NAMESPACE`] before the tag is -/// written: a top-level name or a directory is 405, a foreign tag there or any tag above -/// it 409. `None` when the `PUT` may proceed. -fn put_refusal(store: &Store, segs: &[String], href: &str) -> Result>> { - if object_of(segs).is_none() { - return Ok(Some(method_not_allowed(href, ALLOWED_FILES))); - } - Ok(match resolve(store, segs)? { - Some(Node::Dir(_)) => Some(method_not_allowed(href, ALLOWED_FILES)), - Some(Node::Foreign(_)) => Some(parent_is_object(href)), - _ if under_object(store, segs)? => Some(parent_is_object(href)), - _ => None, - }) -} - /// Stream an object into staging, hashing it on the way, then promote the blob and write the /// raw-file manifest and tag: 201 for a creation, 204 for a replacement. Every refusal but a /// declared length over the cap reads the body through first: a status sent with request @@ -492,27 +322,18 @@ async fn put( // Refused before the body is staged when the answer is already known; checked again // once it is, under the namespace lock. let early = { - let (store, segs, href) = (Arc::clone(&state.store), segs.clone(), href.to_string()); - tokio::task::spawn_blocking(move || put_refusal(&store, &segs, &href)) + let (store, segs) = (Arc::clone(&state.store), segs.clone()); + tokio::task::spawn_blocking(move || crate::files::unwritable(&store, &segs)) .await .context("checking a PUT panicked")?? }; - if let Some(resp) = early { - return refuse(req, resp, href).await; + if let Some(why) = early { + return refuse(req, put_refusal(why, href), href).await; } // Stream uploads to bound memory use; the digest falls out of the same pass. - let (path, file) = state.store.stage_file()?; - let staging = Staging { path, armed: true }; - let mut file = tokio::fs::File::from_std(file); - let written = drain_into(&mut file, req.into_body(), MAX_OBJECT).await; - let flushed = file - .flush() - .await - .map_err(|e| PutError::Failed(anyhow::Error::from(e).context("writing an object"))); - drop(file); - let (size, hex) = match written.and_then(|w| flushed.map(|()| w)) { - Ok(w) => w, + let staged = match crate::files::stage(&state.store, req.into_body()).await { + Ok(staged) => staged, Err(PutError::TooLarge) => { return Ok(error_response( StatusCode::PAYLOAD_TOO_LARGE, @@ -522,42 +343,13 @@ async fn put( } Err(PutError::Failed(e)) => return Err(e), }; - - // Off the runtime: staging compresses, and a large object would otherwise block a tokio - // worker for seconds. The locks are taken inside, after that pass — the store lock as - // the relay takes it. - let store = Arc::clone(&state.store); - let href = href.to_string(); - tokio::task::spawn_blocking(move || -> Result> { - let mut staging = staging; - let staged = store.stage_promotion(&hex, &staging.path)?; - let _namespace = namespace(&store)?; - // shared store lock (vs. an exclusive gc) across the promote and the manifest that - // references it; see Store::lock_shared. A `MKCOL` or another `PUT` may have changed - // the path while the body streamed, so it is checked again. - let checked = store.lock_shared().and_then(|lock| { - let existed = matches!(resolve(&store, &segs)?, Some(Node::Object(_))); - Ok((lock, put_refusal(&store, &segs, &href)?, existed)) - }); - let (_lock, existed) = match checked { - Ok((lock, None, existed)) => (lock, existed), - Ok((_, Some(resp), _)) => { - staged.discard(); - return Ok(resp); - } - Err(e) => { - staged.discard(); - return Err(e); - } - }; - store.promote_staged(&hex, staged)?; - staging.armed = false; - let (repo, tag) = object_of(&segs).context("a PUT past its checks names an object")?; - store.put_raw_file(&repo, tag, &hex, size, None)?; - Ok(if existed { no_content() } else { created() }) - }) - .await - .context("the object's promotion panicked")? + Ok( + match crate::files::commit(Arc::clone(&state.store), segs, staged).await? { + Written::Created => created(), + Written::Replaced => no_content(), + Written::Refused(why) => put_refusal(why, href), + }, + ) } /// Read a request's body through and discard it: a status sent with request bytes unread @@ -589,55 +381,9 @@ fn parent_is_object(href: &str) -> Response { error_response(StatusCode::CONFLICT, "DENIED", href) } -/// Stream `body` to `out`, coalescing small frames and enforcing `cap`, including for chunked -/// bodies; `(bytes written, sha256 hex)` on success. `out` may be a sink when draining a -/// rejected PUT. The generic body allows tests with a small cap. -async fn drain_into( - out: &mut (impl AsyncWrite + Unpin), - mut body: B, - cap: u64, -) -> std::result::Result<(u64, String), PutError> -where - B: hyper::body::Body + Unpin, - B::Error: std::error::Error + Send + Sync + 'static, -{ - let mut buf: Vec = Vec::with_capacity(STREAM_CHUNK); - let mut hasher = Sha256::new(); - let mut total: u64 = 0; - let io = |e: std::io::Error| PutError::Failed(anyhow::Error::from(e).context("writing")); - while let Some(frame) = body.frame().await { - let frame = frame - .map_err(|e| PutError::Failed(anyhow::Error::from(e).context("reading a PUT body")))?; - // Trailers carry no content; a body that has them simply ends after them. - let Ok(data) = frame.into_data() else { - continue; - }; - total = total.saturating_add(data.len() as u64); - if total > cap { - return Err(PutError::TooLarge); - } - hasher.update(&data); - // A frame already worth a write goes straight out rather than through the - // coalescing buffer, which is there for the small ones. - if buf.is_empty() && data.len() >= STREAM_CHUNK { - out.write_all(&data).await.map_err(io)?; - continue; - } - buf.extend_from_slice(&data); - if buf.len() >= STREAM_CHUNK { - out.write_all(&buf).await.map_err(io)?; - buf.clear(); - } - } - if !buf.is_empty() { - out.write_all(&buf).await.map_err(io)?; - } - Ok((total, crate::hex_of(&hasher.finalize()))) -} - /// Create a directory — an empty repository — and its missing ancestors: 201, 405 when the /// path is already mapped (a directory or a tag), or 409 when a tag is above it. Checked and -/// created under [`NAMESPACE`]. +/// created under the [`namespace`] lock. fn mkcol(store: &Store, segs: &[String], href: &str) -> Result> { let _namespace = namespace(store)?; match resolve(store, segs)? { @@ -660,9 +406,9 @@ const DELETE_LOCK_WAIT: Duration = Duration::from_secs(30); /// caller. Recursive deletion is unsupported. An object's bytes stay in the pool until the /// gc finds them unreferenced. /// -/// A directory's removal needs the store lock exclusive. It is tried under [`NAMESPACE`], -/// which is released between attempts so other `files/` writes proceed; past -/// [`DELETE_LOCK_WAIT`] of shared holders or a gc, the answer is 503. +/// A directory's removal needs the store lock exclusive. It is tried under the +/// [`namespace`] lock, which is released between attempts so other `files/` writes proceed; +/// past [`DELETE_LOCK_WAIT`] of shared holders or a gc, the answer is 503. fn delete( store: &Store, authz: &Authz<'_>, @@ -719,86 +465,3 @@ fn delete( return Ok(resp); } } - -#[cfg(test)] -mod tests { - use super::*; - - /// The namespace held here is held against another process's open of the store: the - /// lock file another descriptor cannot take until it is released. - #[test] - fn the_namespace_is_held_against_other_processes() { - use std::os::fd::AsRawFd; - let dir = std::env::temp_dir().join(format!("vk-reg-files-ns-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&dir); - let store = Store::new(dir.clone()).unwrap(); - let try_lock = || { - let file = std::fs::File::open(dir.join(".files.lock")).unwrap(); - // SAFETY: flock on a descriptor `file` owns. - unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) == 0 } - }; - let held = namespace(&store).unwrap(); - assert!(!try_lock()); - drop(held); - assert!(try_lock()); - let _ = std::fs::remove_dir_all(&dir); - } - - /// Verify the streaming size cap with a small limit and multiple frames, and the digest - /// of what went through. - #[tokio::test] - async fn a_body_over_the_cap_stops_being_written() { - let dir = std::env::temp_dir().join(format!("vk-reg-files-cap-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).unwrap(); - let path = dir.join("staged"); - - // Several frames, so the cap is crossed part way through rather than at the first. - let frames = || { - let chunks: Vec, std::io::Error>> = (0 - ..4) - .map(|_| Ok(hyper::body::Frame::data(Bytes::from(vec![1u8; 100])))) - .collect(); - http_body_util::StreamBody::new(futures::stream::iter(chunks)) - }; - - let mut file = tokio::fs::File::create(&path).await.unwrap(); - assert!(matches!( - drain_into(&mut file, frames(), 250).await, - Err(PutError::TooLarge) - )); - drop(file); - // Refused where the count crossed, so the partial file is bounded by the cap plus - // the frame that crossed it — never the whole body. - let staged = std::fs::metadata(&path).unwrap().len(); - assert!(staged <= 300, "wrote {staged} bytes past a 250-byte cap"); - - // Exactly at the cap is not over it, and the digest is the body's. - let mut file = tokio::fs::File::create(&path).await.unwrap(); - let (n, hex) = drain_into(&mut file, frames(), 400).await.ok().unwrap(); - file.flush().await.unwrap(); - drop(file); - assert_eq!(n, 400); - assert_eq!(std::fs::metadata(&path).unwrap().len(), 400); - assert_eq!(hex, crate::sha256_hex_raw(&vec![1u8; 400])); - - let _ = std::fs::remove_dir_all(&dir); - } - - #[test] - fn paths_map_onto_repositories_and_tags() { - let segs = |s: &[&str]| s.iter().map(|s| s.to_string()).collect::>(); - assert_eq!(repo_name(&segs(&[])), "files"); - assert_eq!( - repo_name(&segs(&["sccache", "a", "b"])), - "files/sccache/a/b" - ); - assert!( - object_of(&segs(&["sccache"])).is_none(), - "a top-level name is a directory" - ); - let key = segs(&["sccache", "a", "b", "abcdef"]); - let (repo, tag) = object_of(&key).unwrap(); - assert_eq!((repo.as_str(), tag), ("files/sccache/a/b", "abcdef")); - } -} diff --git a/vk-registry/src/dav/mod.rs b/vk-registry/src/dav/mod.rs index 8690af3c..b22e0be0 100644 --- a/vk-registry/src/dav/mod.rs +++ b/vk-registry/src/dav/mod.rs @@ -90,7 +90,7 @@ async fn root( entries.push(Entry::collection(href(&["repos"], true), repos)); entries.push(Entry::collection( href(&["files"], true), - store.repos_path_modified(files::FILES_REPO), + store.repos_path_modified(crate::files::FILES_REPO), )); } Ok(multistatus(&entries)) diff --git a/vk-registry/src/files.rs b/vk-registry/src/files.rs new file mode 100644 index 00000000..5f885837 --- /dev/null +++ b/vk-registry/src/files.rs @@ -0,0 +1,436 @@ +//! The plain-file area of the pool, `files/`, as objects in directories, below any wire +//! protocol. Directories map to repositories under `files/`; each object is a tag on a +//! single-layer raw-file manifest, as written by `/upload`, whose layer holds the file bytes. +//! A write streams to `uploads/` while hashing ([`stage`]), then promotes the blob and writes +//! the manifest and tag under the namespace lock ([`commit`]). Each wire protocol over this +//! layer keeps its own status codes and bodies: WebDAV under `/dav/files/` (`crate::dav`) +//! today, an S3 view in later work. Entries expire under gc's tag retention and blob grace +//! windows. + +use std::path::PathBuf; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::{Duration, SystemTime}; + +use anyhow::{Context, Result}; +use bytes::Bytes; +use http_body_util::BodyExt; +use sha2::{Digest, Sha256}; +use tokio::io::{AsyncWrite, AsyncWriteExt}; + +use crate::{Authz, STREAM_CHUNK, Store, accounts}; + +/// The repository every object and directory lives under. +pub(crate) const FILES_REPO: &str = "files"; + +/// Maximum object size, checked both against a declared length and while streaming. +pub(crate) const MAX_OBJECT: u64 = 4 << 30; + +/// Minimum idle age before a read refreshes the tag's mtime — the "last used" record the +/// gc's retention keys on — so a cache read is a metadata write at most once an hour. +const TOUCH_AFTER: Duration = Duration::from_secs(3600); + +/// Serialize `files/` checks and namespace changes (object tags, new directories, deletions) +/// in this server so a write of an object and of a directory at the same path cannot both +/// succeed. Acquire before the store lock, never while holding it. `/v2/` writes into +/// `files/` do not take this lock. +static NAMESPACE: Mutex<()> = Mutex::new(()); + +/// The namespace held, in this process and against any other serving the store: a registry +/// and the release it hands over to serve it together for a while. +pub(crate) struct Namespace { + _process: std::sync::MutexGuard<'static, ()>, + _store: crate::LockGuard, +} + +/// Hold [`NAMESPACE`], then the store's namespace lock. Accept a poisoned mutex: it guards +/// no data that a panic could leave inconsistent. +pub(crate) fn namespace(store: &Store) -> Result { + let process = NAMESPACE.lock().unwrap_or_else(PoisonError::into_inner); + Ok(Namespace { + _process: process, + _store: store.lock_namespace()?, + }) +} + +/// What a `files/` path names once the store has been looked at. +pub(crate) enum Node { + /// A repository under `files/`, or a path component above one: a directory. + Dir(String), + /// A tag on a raw-file manifest: an object. + Object(Object), + /// A tag on any other manifest — an image pushed over `/v2/` into a `files/` + /// repository. Absent to reads; a write there is refused, so nothing here replaces or + /// deletes it. + Foreign(String), +} + +/// An object's tag and what its manifest says about the layer. +pub(crate) struct Object { + pub(crate) repo: String, + pub(crate) tag: String, + /// the layer blob's hex + pub(crate) layer: String, + /// the layer's canonical length + pub(crate) size: u64, + /// the tag's mtime + pub(crate) modified: SystemTime, +} + +impl Node { + /// The repository this node authorizes as. + pub(crate) fn repo(&self) -> &str { + match self { + Node::Dir(r) | Node::Foreign(r) => r, + Node::Object(o) => &o.repo, + } + } +} + +/// The repository a run of path components names: `files/`. +pub(crate) fn repo_name(segs: &[String]) -> String { + let mut name = String::from(FILES_REPO); + for s in segs { + name.push('/'); + name.push_str(s); + } + name +} + +/// The repository and tag an object at `segs` would have: the last component is the tag. +pub(crate) fn object_of(segs: &[String]) -> Option<(String, &str)> { + let (leaf, parents) = segs.split_last()?; + (!parents.is_empty()).then(|| (repo_name(parents), leaf.as_str())) +} + +/// What is at `segs`: a directory, an object, a foreign tag, or nothing. A directory is any +/// path under `repos/files/` that is one — a repository, or the parent a nested repository's +/// name created — so the tree reads as the client wrote it. +pub(crate) fn resolve(store: &Store, segs: &[String]) -> Result> { + let dir = repo_name(segs); + if store.repo_dir_exists(&dir) { + return Ok(Some(Node::Dir(dir))); + } + let Some((repo, tag)) = object_of(segs) else { + return Ok(None); + }; + let Some((manifest_hex, modified)) = store.tag_target(&repo, tag) else { + return Ok(None); + }; + let layer = match store.get_blob(&manifest_hex)? { + Some(manifest) => crate::raw_file_layer(&manifest), + None => None, + }; + Ok(Some(match layer { + Some((layer, size)) => Node::Object(Object { + repo, + tag: tag.to_string(), + layer, + size, + modified, + }), + None => Node::Foreign(repo), + })) +} + +/// Whether an ancestor of `segs` is a tag. Writing below one would make its path a directory +/// too, and [`resolve`] would then find the directory and hide the tag. A top-level name is +/// always a directory. +pub(crate) fn under_object(store: &Store, segs: &[String]) -> Result { + for n in 2..segs.len() { + if matches!( + resolve(store, &segs[..n])?, + Some(Node::Object(_) | Node::Foreign(_)) + ) { + return Ok(true); + } + } + Ok(false) +} + +/// Whether this caller may `action` on `repo`. +pub(crate) fn allowed(authz: &Authz<'_>, action: accounts::Action, repo: &str) -> bool { + crate::authorize_or_forbidden(authz, action, repo).is_none() +} + +/// Refresh the tag of an object just served once it has been idle past [`TOUCH_AFTER`]: a +/// read is a use, which is what keeps the tag from the gc's retention, and a hot entry is +/// not a metadata write per hit. +pub(crate) fn touch_if_idle(store: &Store, obj: &Object) { + if SystemTime::now() + .duration_since(obj.modified) + .is_ok_and(|idle| idle > TOUCH_AFTER) + { + store.touch_tag(&obj.repo, &obj.tag); + } +} + +/// Why `segs` cannot take an object. +pub(crate) enum Unwritable { + /// a single component: a top-level name is always a directory + TopLevel, + /// a directory is there + Directory, + /// a foreign tag is there + Foreign, + /// a tag is above it + UnderObject, +} + +/// Why `segs` cannot take an object, or `None` when a write may proceed. Checked before a +/// body is staged when the answer is already known, and again by [`commit`] under +/// [`NAMESPACE`] before the tag is written. +pub(crate) fn unwritable(store: &Store, segs: &[String]) -> Result> { + if object_of(segs).is_none() { + return Ok(Some(Unwritable::TopLevel)); + } + Ok(match resolve(store, segs)? { + Some(Node::Dir(_)) => Some(Unwritable::Directory), + Some(Node::Foreign(_)) => Some(Unwritable::Foreign), + _ if under_object(store, segs)? => Some(Unwritable::UnderObject), + _ => None, + }) +} + +/// Why a streamed body stopped short. +pub(crate) enum PutError { + /// the body crossed the cap — decided while reading + TooLarge, + /// the read or the write failed + Failed(anyhow::Error), +} + +/// A staging file under `uploads/`, unlinked when dropped unless its bytes were promoted — +/// including when the handler's future is dropped with the connection mid-body. +struct Staging { + path: PathBuf, + armed: bool, +} + +impl Drop for Staging { + fn drop(&mut self) { + if self.armed { + // Gone already (a promotion that renamed it, or a discard) is the goal; any + // other failure leaves it to the gc's sweep of idle uploads. + let _ = std::fs::remove_file(&self.path); + } + } +} + +/// A body written whole to a staging file and hashed, not yet in the pool. Dropping it +/// removes the file, so a caller that refuses what it sees here simply lets it go. +pub(crate) struct Staged { + /// the body's length + pub(crate) size: u64, + /// the body's sha256 hex + pub(crate) hex: String, + staging: Staging, +} + +/// Stream `body` into a staging file under `uploads/`, hashing it on the way, up to +/// [`MAX_OBJECT`]. Memory use is bounded by the coalescing buffer whatever the size. +pub(crate) async fn stage(store: &Store, body: B) -> std::result::Result +where + B: hyper::body::Body + Unpin, + B::Error: std::error::Error + Send + Sync + 'static, +{ + let (path, file) = store.stage_file().map_err(PutError::Failed)?; + let staging = Staging { path, armed: true }; + let mut file = tokio::fs::File::from_std(file); + let written = drain_into(&mut file, body, MAX_OBJECT).await; + let flushed = file + .flush() + .await + .map_err(|e| PutError::Failed(anyhow::Error::from(e).context("writing an object"))); + drop(file); + let (size, hex) = written.and_then(|w| flushed.map(|()| w))?; + Ok(Staged { size, hex, staging }) +} + +/// What [`commit`] did. +pub(crate) enum Written { + /// the object is new + Created, + /// the object replaced one at the same path + Replaced, + /// the path changed while the body streamed and no longer takes an object; nothing was + /// written + Refused(Unwritable), +} + +/// Promote a [`stage`]d body into the pool and tag it at `segs` as a raw file, re-checking +/// [`unwritable`] under [`NAMESPACE`] and the shared store lock first. Off the runtime: +/// staging compresses, and a large object would otherwise block a tokio worker for seconds. +/// The locks are taken inside, after that pass — the store lock as the relay takes it. +pub(crate) async fn commit( + store: Arc, + segs: Vec, + staged: Staged, +) -> Result { + tokio::task::spawn_blocking(move || -> Result { + let Staged { + size, + hex, + mut staging, + } = staged; + let promotion = store.stage_promotion(&hex, &staging.path)?; + let _namespace = namespace(&store)?; + // shared store lock (vs. an exclusive gc) across the promote and the manifest that + // references it; see Store::lock_shared. Another write may have changed the path + // while the body streamed, so it is checked again. + let checked = store.lock_shared().and_then(|lock| { + let existed = matches!(resolve(&store, &segs)?, Some(Node::Object(_))); + Ok((lock, unwritable(&store, &segs)?, existed)) + }); + let (_lock, existed) = match checked { + Ok((lock, None, existed)) => (lock, existed), + Ok((_, Some(why), _)) => { + promotion.discard(); + return Ok(Written::Refused(why)); + } + Err(e) => { + promotion.discard(); + return Err(e); + } + }; + store.promote_staged(&hex, promotion)?; + staging.armed = false; + let (repo, tag) = object_of(&segs).context("a write past its checks names an object")?; + store.put_raw_file(&repo, tag, &hex, size, None)?; + Ok(if existed { + Written::Replaced + } else { + Written::Created + }) + }) + .await + .context("the object's promotion panicked")? +} + +/// Stream `body` to `out`, coalescing small frames and enforcing `cap`, including for chunked +/// bodies; `(bytes written, sha256 hex)` on success. `out` may be a sink when draining a +/// refused request. The generic body allows tests with a small cap. +pub(crate) async fn drain_into( + out: &mut (impl AsyncWrite + Unpin), + mut body: B, + cap: u64, +) -> std::result::Result<(u64, String), PutError> +where + B: hyper::body::Body + Unpin, + B::Error: std::error::Error + Send + Sync + 'static, +{ + let mut buf: Vec = Vec::with_capacity(STREAM_CHUNK); + let mut hasher = Sha256::new(); + let mut total: u64 = 0; + let io = |e: std::io::Error| PutError::Failed(anyhow::Error::from(e).context("writing")); + while let Some(frame) = body.frame().await { + let frame = frame + .map_err(|e| PutError::Failed(anyhow::Error::from(e).context("reading a PUT body")))?; + // Trailers carry no content; a body that has them simply ends after them. + let Ok(data) = frame.into_data() else { + continue; + }; + total = total.saturating_add(data.len() as u64); + if total > cap { + return Err(PutError::TooLarge); + } + hasher.update(&data); + // A frame already worth a write goes straight out rather than through the + // coalescing buffer, which is there for the small ones. + if buf.is_empty() && data.len() >= STREAM_CHUNK { + out.write_all(&data).await.map_err(io)?; + continue; + } + buf.extend_from_slice(&data); + if buf.len() >= STREAM_CHUNK { + out.write_all(&buf).await.map_err(io)?; + buf.clear(); + } + } + if !buf.is_empty() { + out.write_all(&buf).await.map_err(io)?; + } + Ok((total, crate::hex_of(&hasher.finalize()))) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The namespace held here is held against another process's open of the store: the + /// lock file another descriptor cannot take until it is released. + #[test] + fn the_namespace_is_held_against_other_processes() { + use std::os::fd::AsRawFd; + let dir = std::env::temp_dir().join(format!("vk-reg-files-ns-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let store = Store::new(dir.clone()).unwrap(); + let try_lock = || { + let file = std::fs::File::open(dir.join(".files.lock")).unwrap(); + // SAFETY: flock on a descriptor `file` owns. + unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) == 0 } + }; + let held = namespace(&store).unwrap(); + assert!(!try_lock()); + drop(held); + assert!(try_lock()); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Verify the streaming size cap with a small limit and multiple frames, and the digest + /// of what went through. + #[tokio::test] + async fn a_body_over_the_cap_stops_being_written() { + let dir = std::env::temp_dir().join(format!("vk-reg-files-cap-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("staged"); + + // Several frames, so the cap is crossed part way through rather than at the first. + let frames = || { + let chunks: Vec, std::io::Error>> = (0 + ..4) + .map(|_| Ok(hyper::body::Frame::data(Bytes::from(vec![1u8; 100])))) + .collect(); + http_body_util::StreamBody::new(futures::stream::iter(chunks)) + }; + + let mut file = tokio::fs::File::create(&path).await.unwrap(); + assert!(matches!( + drain_into(&mut file, frames(), 250).await, + Err(PutError::TooLarge) + )); + drop(file); + // Refused where the count crossed, so the partial file is bounded by the cap plus + // the frame that crossed it — never the whole body. + let staged = std::fs::metadata(&path).unwrap().len(); + assert!(staged <= 300, "wrote {staged} bytes past a 250-byte cap"); + + // Exactly at the cap is not over it, and the digest is the body's. + let mut file = tokio::fs::File::create(&path).await.unwrap(); + let (n, hex) = drain_into(&mut file, frames(), 400).await.ok().unwrap(); + file.flush().await.unwrap(); + drop(file); + assert_eq!(n, 400); + assert_eq!(std::fs::metadata(&path).unwrap().len(), 400); + assert_eq!(hex, crate::sha256_hex_raw(&vec![1u8; 400])); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn paths_map_onto_repositories_and_tags() { + let segs = |s: &[&str]| s.iter().map(|s| s.to_string()).collect::>(); + assert_eq!(repo_name(&segs(&[])), "files"); + assert_eq!( + repo_name(&segs(&["sccache", "a", "b"])), + "files/sccache/a/b" + ); + assert!( + object_of(&segs(&["sccache"])).is_none(), + "a top-level name is a directory" + ); + let key = segs(&["sccache", "a", "b", "abcdef"]); + let (repo, tag) = object_of(&key).unwrap(); + assert_eq!((repo.as_str(), tag), ("files/sccache/a/b", "abcdef")); + } +} diff --git a/vk-registry/src/lib.rs b/vk-registry/src/lib.rs index 13553873..ae226434 100644 --- a/vk-registry/src/lib.rs +++ b/vk-registry/src/lib.rs @@ -51,6 +51,7 @@ pub(crate) mod captions; pub mod client; pub mod config; pub(crate) mod dav; +pub(crate) mod files; pub(crate) mod forms; pub mod handover; pub(crate) mod html; From acff68c58a90a75c6122d34b0639c9f207438acc Mon Sep 17 00:00:00 2001 From: Antoine Bernardeau Date: Thu, 5 Nov 2026 09:49:24 +0100 Subject: [PATCH 02/54] vk-registry: add the s3 server setting --- vk-driver/src/registry.rs | 8 +++++++ vk-registry/src/config.rs | 35 +++++++++++++++++++++++++++++++ vk-registry/src/config/help.rs | 5 +++++ vk-registry/src/hub/mod.rs | 1 + vk-registry/src/lib.rs | 7 +++++++ vk-registry/tests/accounts_e2e.rs | 4 ++++ vk-registry/tests/exists_e2e.rs | 1 + vk-registry/tests/hubtoken_e2e.rs | 1 + vk-registry/tests/relay_e2e.rs | 16 ++++++++++++++ 9 files changed, 78 insertions(+) diff --git a/vk-driver/src/registry.rs b/vk-driver/src/registry.rs index f3999db4..19a3fce2 100644 --- a/vk-driver/src/registry.rs +++ b/vk-driver/src/registry.rs @@ -5660,6 +5660,7 @@ pub(crate) mod tests { auth: vk_registry::Authenticator::Shared(auth), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), })) @@ -5707,6 +5708,7 @@ pub(crate) mod tests { auth: vk_registry::Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), })); @@ -5742,6 +5744,7 @@ pub(crate) mod tests { auth: vk_registry::Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), })); @@ -5855,6 +5858,7 @@ pub(crate) mod tests { auth: vk_registry::Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), })); @@ -6104,6 +6108,7 @@ pub(crate) mod tests { auth: vk_registry::Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), })); @@ -6162,6 +6167,7 @@ pub(crate) mod tests { auth: vk_registry::Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), })); @@ -6203,6 +6209,7 @@ pub(crate) mod tests { auth: vk_registry::Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), })); @@ -6239,6 +6246,7 @@ pub(crate) mod tests { auth: vk_registry::Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), })); diff --git a/vk-registry/src/config.rs b/vk-registry/src/config.rs index 78b639e7..461a241a 100644 --- a/vk-registry/src/config.rs +++ b/vk-registry/src/config.rs @@ -87,6 +87,9 @@ pub struct ServerConfig { /// objects already stored under `files/` stay reachable over `/v2/` and expire under `gc` /// like any tag. pub webdav: bool, + /// Enable `/s3/` (default: true). When disabled, requests return 404 after auth; the + /// objects already stored under `files/` stay reachable over `/dav/` and `/v2/`. + pub s3: bool, /// Connections served at once; past it, new ones wait in the listen backlog. pub max_connections: usize, /// Connections served at once from one client address (an IPv6 /64); past it, new ones @@ -258,6 +261,8 @@ struct FileConfig { admin_socket: Option, /// Enable `/dav/`; defaults to true. webdav: Option, + /// Enable `/s3/`; defaults to true. + s3: Option, max_connections: Option, max_connections_per_client: Option, oidc: Option, @@ -378,6 +383,7 @@ impl ServerConfig { admin_socket: AdminSocket::Unset, oidc: None, webdav: true, + s3: true, max_connections: crate::DEFAULT_MAX_CONNECTIONS, max_connections_per_client: crate::DEFAULT_MAX_CONNECTIONS_PER_CLIENT, hub: HubSettings::default(), @@ -452,6 +458,10 @@ impl ServerConfig { if *tokens { described.push_str(" tokens=true"); } + // Only when off, so a registry that leaves it on keeps the hash it had. + if !self.s3 { + described.push_str(" s3=false"); + } // Only when set, so a registry that leaves it keeps the hash it had. if *update_drain_minutes != DEFAULT_UPDATE_DRAIN_MINUTES { described.push_str(&format!(" update_drain_minutes={update_drain_minutes}")); @@ -669,6 +679,7 @@ impl ServerConfig { public_url: o.public_url, }), webdav: f.webdav.unwrap_or(true), + s3: f.s3.unwrap_or(true), max_connections: f.max_connections.unwrap_or(crate::DEFAULT_MAX_CONNECTIONS), max_connections_per_client: f .max_connections_per_client @@ -952,6 +963,7 @@ impl ServerConfig { auth, tls: None, webdav: self.webdav, + s3: self.s3, tokens: Default::default(), handover: Default::default(), }) @@ -1889,6 +1901,29 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + /// `s3` defaults to on, reads both ways, and changes the config hash only when off. + #[test] + fn s3_defaults_to_on_and_changes_the_config_hash_only_when_off() { + let dir = std::env::temp_dir().join(format!("vk-regserve-s3-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let load = |body: &str| { + let p = dir.join("c.toml"); + std::fs::write(&p, body).unwrap(); + ServerConfig::load(&p, None, None).unwrap() + }; + assert!(load("").s3); + assert!(load("s3 = true\n").s3); + let off = load("s3 = false\n"); + assert!(!off.s3); + let mut cfg = ServerConfig::local("127.0.0.1:5000".parse().unwrap(), "/s".into()); + assert!(cfg.s3); + assert_eq!(cfg.hub_facts().config_hash, "e3609a2059e9c15e"); + cfg.s3 = false; + assert_ne!(cfg.hub_facts().config_hash, "e3609a2059e9c15e"); + let _ = std::fs::remove_dir_all(&dir); + } + /// With `[hub] tokens = true`, a shared secret that reads as a hub token is refused. #[test] fn a_shared_secret_that_reads_as_a_hub_token_is_refused_with_tokens_on() { diff --git a/vk-registry/src/config/help.rs b/vk-registry/src/config/help.rs index 5c2b81d8..2acb29da 100644 --- a/vk-registry/src/config/help.rs +++ b/vk-registry/src/config/help.rs @@ -96,6 +96,11 @@ const KEYS: &[Key] = &[ name: "webdav", help: "enable /dav/; false returns 404 [default: true]", }, + Key { + table: Table::Top, + name: "s3", + help: "enable /s3/; false returns 404 [default: true]", + }, Key { table: Table::Top, name: "max_connections", diff --git a/vk-registry/src/hub/mod.rs b/vk-registry/src/hub/mod.rs index b699d9c0..8e114638 100644 --- a/vk-registry/src/hub/mod.rs +++ b/vk-registry/src/hub/mod.rs @@ -921,6 +921,7 @@ mod tests { auth: crate::Authenticator::Shared(crate::auth::Auth::None), tls: None, webdav: false, + s3: false, tokens: Default::default(), handover: Default::default(), }) diff --git a/vk-registry/src/lib.rs b/vk-registry/src/lib.rs index ae226434..b8946db9 100644 --- a/vk-registry/src/lib.rs +++ b/vk-registry/src/lib.rs @@ -107,6 +107,8 @@ pub struct ServerState { pub tls: Option, /// Enable WebDAV routes. pub webdav: bool, + /// Enable the S3 routes under `/s3/`. + pub s3: bool, /// The hub tokens taken, once enrolled with `[hub] tokens = true`. pub tokens: hubtoken::TokenGate, /// The listeners served through, to hand to a release. @@ -4042,6 +4044,9 @@ async fn serve_refusing( if !state.webdav { eprintln!("vk-registry: WebDAV off (webdav = false): /dav/ answers 404"); } + if !state.s3 { + eprintln!("vk-registry: S3 off (s3 = false): /s3/ answers 404"); + } let slots = Arc::new(tokio::sync::Semaphore::new(limits.connections)); let peers = PeerCounts::default(); let mut refused = RefusalLog::default(); @@ -6414,6 +6419,7 @@ mod tests { auth: Authenticator::Shared(auth), tls, webdav: false, + s3: false, tokens: Default::default(), handover: Default::default(), }); @@ -6855,6 +6861,7 @@ mod tests { auth: Authenticator::Shared(auth::Auth::None), tls: acceptor, webdav: false, + s3: false, tokens: Default::default(), handover: Default::default(), }); diff --git a/vk-registry/tests/accounts_e2e.rs b/vk-registry/tests/accounts_e2e.rs index 2425ace7..48b8e6a4 100644 --- a/vk-registry/tests/accounts_e2e.rs +++ b/vk-registry/tests/accounts_e2e.rs @@ -1366,6 +1366,7 @@ async fn a_relayed_blob_becomes_a_member_of_the_repo_it_was_fetched_for() { auth: Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), })); @@ -1471,6 +1472,7 @@ async fn shared_secret_mode_is_unchanged_by_repo_scoping() { auth: Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), })); @@ -1691,6 +1693,7 @@ async fn deleting_a_manifest_on_an_open_server() { auth: Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), })); @@ -2185,6 +2188,7 @@ async fn a_cross_site_write_is_refused_on_an_open_server() { auth: Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), })); diff --git a/vk-registry/tests/exists_e2e.rs b/vk-registry/tests/exists_e2e.rs index dbd95932..1a5f4fb8 100644 --- a/vk-registry/tests/exists_e2e.rs +++ b/vk-registry/tests/exists_e2e.rs @@ -61,6 +61,7 @@ async fn batch_probe_answers_every_tag_in_order() { auth: vk_registry::Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), })); diff --git a/vk-registry/tests/hubtoken_e2e.rs b/vk-registry/tests/hubtoken_e2e.rs index 3adfa24e..3dc11cac 100644 --- a/vk-registry/tests/hubtoken_e2e.rs +++ b/vk-registry/tests/hubtoken_e2e.rs @@ -90,6 +90,7 @@ fn shared_with( auth: vk_registry::Authenticator::Shared(auth), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), }); diff --git a/vk-registry/tests/relay_e2e.rs b/vk-registry/tests/relay_e2e.rs index 2b8d2ec4..7090e158 100644 --- a/vk-registry/tests/relay_e2e.rs +++ b/vk-registry/tests/relay_e2e.rs @@ -96,6 +96,7 @@ async fn relay_caches_digest_not_tag() { auth: vk_registry::Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), }); @@ -117,6 +118,7 @@ async fn relay_caches_digest_not_tag() { auth: vk_registry::Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), }); @@ -191,6 +193,7 @@ async fn multi_lock_is_atomic_all_or_nothing() { auth: vk_registry::Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), }); @@ -267,6 +270,7 @@ async fn lock_client_authenticates_with_basic() { }), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), }); @@ -311,6 +315,7 @@ async fn lock_client_authenticates_with_bearer() { }), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), }); @@ -351,6 +356,7 @@ async fn lock_client_round_trips_against_the_server() { auth: vk_registry::Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), }); @@ -420,6 +426,7 @@ async fn bearer_auth_gates_everything_including_the_probe() { }), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), }); @@ -505,6 +512,7 @@ async fn basic_auth_gates_and_challenges() { }), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), }); @@ -722,6 +730,7 @@ async fn relay_does_not_leak_upstream_credentials_to_the_client() { auth: vk_registry::Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), }); @@ -766,6 +775,7 @@ async fn lock_api_build_once_over_http() { auth: vk_registry::Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), }); @@ -1075,6 +1085,7 @@ async fn browse_belongs_to_accounts_mode_and_redirects_a_signed_out_browser() { auth: vk_registry::Authenticator::Shared(vk_registry::auth::Auth::None), tls: None, webdav: true, + s3: true, tokens: Default::default(), handover: Default::default(), }); @@ -1267,6 +1278,7 @@ fn mirror_of(up_addr: std::net::SocketAddr, dir: &std::path::Path) -> Arc Date: Thu, 5 Nov 2026 10:29:02 +0100 Subject: [PATCH 03/54] vk-registry/s3: check SigV4 credentials on /s3/ Requests under /s3/ now pass a gate S3 clients understand: a SigV4 Authorization header is verified, refusals are S3 XML errors carrying a request id, and a missing or refused credential is a 403 rather than the 401 challenge. The access key id is a credential the registry already takes (an API key, a hub token, the shared bearer, or the Basic user), authenticated as a bearer would be before any secret is used, and the signing secret is that same string (the password for the Basic user), so nothing new is stored. Presigned URLs and temporary credentials are refused. No operation is served yet: authenticated requests get 404 or 501. --- Cargo.lock | 2 + NOTICE | 6 + vk-registry/Cargo.toml | 3 + vk-registry/src/accounts.rs | 9 +- vk-registry/src/auth.rs | 40 +- vk-registry/src/dav/mod.rs | 2 +- vk-registry/src/files.rs | 14 + vk-registry/src/lib.rs | 15 +- vk-registry/src/s3/mod.rs | 443 +++++++++ vk-registry/src/s3/sigv4.rs | 879 ++++++++++++++++++ vk-registry/src/s3/testdata/sigv4/LICENSE | 202 ++++ vk-registry/src/s3/testdata/sigv4/README | 14 + .../get-header-key-duplicate/context.json | 12 + .../header-canonical-request.txt | 9 + .../header-signature.txt | 1 + .../header-signed-request.txt | 8 + .../header-string-to-sign.txt | 4 + .../get-header-key-duplicate/request.txt | 5 + .../get-header-value-multiline/context.json | 12 + .../header-canonical-request.txt | 9 + .../header-signature.txt | 1 + .../header-signed-request.txt | 8 + .../header-string-to-sign.txt | 4 + .../get-header-value-multiline/request.txt | 5 + .../sigv4/get-header-value-order/context.json | 12 + .../header-canonical-request.txt | 9 + .../header-signature.txt | 1 + .../header-signed-request.txt | 9 + .../header-string-to-sign.txt | 4 + .../sigv4/get-header-value-order/request.txt | 6 + .../sigv4/get-header-value-trim/context.json | 12 + .../header-canonical-request.txt | 10 + .../header-signature.txt | 1 + .../header-signed-request.txt | 7 + .../header-string-to-sign.txt | 4 + .../sigv4/get-header-value-trim/request.txt | 4 + .../get-relative-normalized/context.json | 12 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../sigv4/get-relative-normalized/request.txt | 2 + .../context.json | 12 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../request.txt | 2 + .../context.json | 12 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../request.txt | 2 + .../get-relative-unnormalized/context.json | 12 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../get-relative-unnormalized/request.txt | 2 + .../context.json | 12 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../request.txt | 2 + .../context.json | 12 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../request.txt | 2 + .../sigv4/get-slash-normalized/context.json | 12 + .../header-canonical-request.txt | 8 + .../get-slash-normalized/header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../sigv4/get-slash-normalized/request.txt | 2 + .../context.json | 12 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../request.txt | 2 + .../context.json | 12 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../request.txt | 2 + .../sigv4/get-slash-unnormalized/context.json | 12 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../sigv4/get-slash-unnormalized/request.txt | 2 + .../sigv4/get-slashes-normalized/context.json | 12 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../sigv4/get-slashes-normalized/request.txt | 2 + .../get-slashes-unnormalized/context.json | 12 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../get-slashes-unnormalized/request.txt | 2 + .../sigv4/get-space-normalized/context.json | 12 + .../header-canonical-request.txt | 8 + .../get-space-normalized/header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../sigv4/get-space-normalized/request.txt | 2 + .../sigv4/get-space-unnormalized/context.json | 12 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../sigv4/get-space-unnormalized/request.txt | 2 + .../sigv4/get-unreserved/context.json | 12 + .../header-canonical-request.txt | 8 + .../sigv4/get-unreserved/header-signature.txt | 1 + .../get-unreserved/header-signed-request.txt | 5 + .../get-unreserved/header-string-to-sign.txt | 4 + .../testdata/sigv4/get-unreserved/request.txt | 2 + .../s3/testdata/sigv4/get-utf8/context.json | 12 + .../get-utf8/header-canonical-request.txt | 8 + .../sigv4/get-utf8/header-signature.txt | 1 + .../sigv4/get-utf8/header-signed-request.txt | 5 + .../sigv4/get-utf8/header-string-to-sign.txt | 4 + .../s3/testdata/sigv4/get-utf8/request.txt | 2 + .../get-vanilla-empty-query-key/context.json | 12 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../get-vanilla-empty-query-key/request.txt | 2 + .../context.json | 12 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../request.txt | 2 + .../context.json | 12 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../request.txt | 2 + .../get-vanilla-query-unreserved/context.json | 12 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../get-vanilla-query-unreserved/request.txt | 2 + .../sigv4/get-vanilla-query/context.json | 12 + .../header-canonical-request.txt | 8 + .../get-vanilla-query/header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../sigv4/get-vanilla-query/request.txt | 2 + .../sigv4/get-vanilla-utf8-query/context.json | 12 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../sigv4/get-vanilla-utf8-query/request.txt | 2 + .../context.json | 13 + .../header-canonical-request.txt | 9 + .../header-signature.txt | 1 + .../header-signed-request.txt | 6 + .../header-string-to-sign.txt | 4 + .../request.txt | 2 + .../testdata/sigv4/get-vanilla/context.json | 12 + .../get-vanilla/header-canonical-request.txt | 8 + .../sigv4/get-vanilla/header-signature.txt | 1 + .../get-vanilla/header-signed-request.txt | 5 + .../get-vanilla/header-string-to-sign.txt | 4 + .../s3/testdata/sigv4/get-vanilla/request.txt | 2 + .../sigv4/post-header-key-case/context.json | 12 + .../header-canonical-request.txt | 8 + .../post-header-key-case/header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../sigv4/post-header-key-case/request.txt | 2 + .../sigv4/post-header-key-sort/context.json | 12 + .../header-canonical-request.txt | 9 + .../post-header-key-sort/header-signature.txt | 1 + .../header-signed-request.txt | 6 + .../header-string-to-sign.txt | 4 + .../sigv4/post-header-key-sort/request.txt | 3 + .../sigv4/post-header-value-case/context.json | 12 + .../header-canonical-request.txt | 9 + .../header-signature.txt | 1 + .../header-signed-request.txt | 6 + .../header-string-to-sign.txt | 4 + .../sigv4/post-header-value-case/request.txt | 3 + .../sigv4/post-sts-header-after/context.json | 14 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 6 + .../header-string-to-sign.txt | 4 + .../sigv4/post-sts-header-after/request.txt | 2 + .../sigv4/post-sts-header-before/context.json | 14 + .../header-canonical-request.txt | 9 + .../header-signature.txt | 1 + .../header-signed-request.txt | 6 + .../header-string-to-sign.txt | 4 + .../sigv4/post-sts-header-before/request.txt | 2 + .../context.json | 12 + .../header-canonical-request.txt | 8 + .../header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../request.txt | 2 + .../sigv4/post-vanilla-query/context.json | 12 + .../header-canonical-request.txt | 8 + .../post-vanilla-query/header-signature.txt | 1 + .../header-signed-request.txt | 5 + .../header-string-to-sign.txt | 4 + .../sigv4/post-vanilla-query/request.txt | 2 + .../testdata/sigv4/post-vanilla/context.json | 12 + .../post-vanilla/header-canonical-request.txt | 8 + .../sigv4/post-vanilla/header-signature.txt | 1 + .../post-vanilla/header-signed-request.txt | 5 + .../post-vanilla/header-string-to-sign.txt | 4 + .../testdata/sigv4/post-vanilla/request.txt | 2 + .../context.json | 12 + .../header-canonical-request.txt | 11 + .../header-signature.txt | 1 + .../header-signed-request.txt | 9 + .../header-string-to-sign.txt | 4 + .../request.txt | 6 + .../post-x-www-form-urlencoded/context.json | 12 + .../header-canonical-request.txt | 11 + .../header-signature.txt | 1 + .../header-signed-request.txt | 9 + .../header-string-to-sign.txt | 4 + .../post-x-www-form-urlencoded/request.txt | 6 + vk-registry/src/s3/xml.rs | 74 ++ vk-registry/tests/s3_e2e.rs | 809 ++++++++++++++++ 242 files changed, 3781 insertions(+), 14 deletions(-) create mode 100644 vk-registry/src/s3/mod.rs create mode 100644 vk-registry/src/s3/sigv4.rs create mode 100644 vk-registry/src/s3/testdata/sigv4/LICENSE create mode 100644 vk-registry/src/s3/testdata/sigv4/README create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-value-order/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-value-order/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-value-order/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-value-order/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-value-order/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-value-order/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-value-trim/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-value-trim/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-value-trim/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-value-trim/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-value-trim/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-header-value-trim/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-normalized/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-normalized/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-normalized/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-normalized/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-normalized/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-normalized/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-normalized/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-normalized/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-normalized/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-normalized/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-normalized/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-normalized/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-space-normalized/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-space-normalized/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-space-normalized/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-space-normalized/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-space-normalized/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-space-normalized/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-unreserved/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-unreserved/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-unreserved/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-unreserved/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-unreserved/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-unreserved/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-utf8/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-utf8/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-utf8/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-utf8/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-utf8/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-utf8/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-query/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/get-vanilla/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-header-key-case/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/post-header-key-case/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-header-key-case/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-header-key-case/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-header-key-case/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-header-key-case/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-header-key-sort/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/post-header-key-sort/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-header-key-sort/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-header-key-sort/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-header-key-sort/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-header-key-sort/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-header-value-case/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/post-header-value-case/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-header-value-case/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-header-value-case/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-header-value-case/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-header-value-case/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-sts-header-after/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/post-sts-header-after/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-sts-header-after/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-sts-header-after/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-sts-header-after/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-sts-header-after/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-sts-header-before/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/post-sts-header-before/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-sts-header-before/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-sts-header-before/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-sts-header-before/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-sts-header-before/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-vanilla-query/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/post-vanilla-query/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-vanilla-query/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-vanilla-query/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-vanilla-query/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-vanilla-query/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-vanilla/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/post-vanilla/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-vanilla/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-vanilla/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-vanilla/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-vanilla/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/context.json create mode 100644 vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/header-canonical-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/header-signature.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/header-signed-request.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/header-string-to-sign.txt create mode 100644 vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/request.txt create mode 100644 vk-registry/src/s3/xml.rs create mode 100644 vk-registry/tests/s3_e2e.rs diff --git a/Cargo.lock b/Cargo.lock index bcc53cc9..096ef156 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4305,6 +4305,7 @@ dependencies = [ "bytes", "clap", "futures", + "hmac", "http-body-util", "httpdate", "hyper", @@ -4328,6 +4329,7 @@ dependencies = [ "vk-oidc", "vk-peer", "vk-selfupdate", + "zeroize", "zstd", ] diff --git a/NOTICE b/NOTICE index cf283b38..dacd4dbb 100644 --- a/NOTICE +++ b/NOTICE @@ -74,6 +74,12 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +vk-registry's SigV4 tests use AWS's signing test suite, as vendored in awslabs/aws-c-auth +(tests/aws-signing-test-suite/v4), copied under vk-registry/src/s3/testdata/sigv4/: +https://github.com/awslabs/aws-c-auth/tree/3281f8692e6fd10562c4585a4dded5c16b322698/tests/aws-signing-test-suite/v4 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +Apache License 2.0; see vk-registry/src/s3/testdata/sigv4/LICENSE. + The variable expansion in vk-gitlab/src/spec.rs (`expand`) and vk-driver/src/node/jobs/vars.rs is adapted from Go's os.Expand (src/os/env.go) and its tests, as gitlab-runner uses it: https://github.com/golang/go/blob/go1.24.0/src/os/env.go diff --git a/vk-registry/Cargo.toml b/vk-registry/Cargo.toml index 225338b2..894afe18 100644 --- a/vk-registry/Cargo.toml +++ b/vk-registry/Cargo.toml @@ -35,6 +35,9 @@ rustls.workspace = true tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "logging", "tls12"] } oci-client.workspace = true sha2.workspace = true +# SigV4's HMAC chain for /s3/; zeroize wipes the keys it derives. +hmac.workspace = true +zeroize.workspace = true zstd.workspace = true libc.workspace = true # Private admin-socket binding, shared with vk-core. diff --git a/vk-registry/src/accounts.rs b/vk-registry/src/accounts.rs index 7fd1a4d5..ac2c1549 100644 --- a/vk-registry/src/accounts.rs +++ b/vk-registry/src/accounts.rs @@ -960,8 +960,13 @@ pub fn resolve_principal( } /// [`resolve_principal`]'s body, over the headers alone — a `Request` can only -/// be built by a live connection, so this is the testable half. -fn resolve_headers(db: &Db, headers: &HeaderMap, secure: bool) -> Result> { +/// be built by a live connection, so this is the testable half. The S3 gate also hands it +/// a SigV4 access key id, as the one header `Authorization: Bearer `. +pub(crate) fn resolve_headers( + db: &Db, + headers: &HeaderMap, + secure: bool, +) -> Result> { // An `Authorization` header is a claim to be authenticated by it, so one that does not // carry a `vkr_…` key is a failed authentication rather than a reason to fall back to // the cookie — otherwise a stale shared-secret client would silently authenticate as diff --git a/vk-registry/src/auth.rs b/vk-registry/src/auth.rs index a0d46e02..fbda55d7 100644 --- a/vk-registry/src/auth.rs +++ b/vk-registry/src/auth.rs @@ -4,8 +4,7 @@ //! `GET /v2/` version probe — its 401 + `WWW-Authenticate` is how OCI clients discover //! they must authenticate before the real blob requests. -use hyper::body::Incoming; -use hyper::{Request, Response}; +use hyper::{HeaderMap, Response}; use crate::Body; @@ -26,14 +25,15 @@ impl Auth { !matches!(self, Auth::None) } - /// Whether `req` carries valid credentials (always true when disabled). - pub fn allows(&self, req: &Request) -> bool { + /// Whether a request with `headers` carries valid credentials (always true when + /// disabled). + pub fn allows(&self, headers: &HeaderMap) -> bool { match self { Auth::None => true, - Auth::Bearer { token } => authorization(req) + Auth::Bearer { token } => authorization(headers) .and_then(|v| v.strip_prefix("Bearer ")) .is_some_and(|t| constant_eq(t.as_bytes(), token.as_bytes())), - Auth::Basic { user, pass } => authorization(req) + Auth::Basic { user, pass } => authorization(headers) .and_then(|v| v.strip_prefix("Basic ")) .and_then(|b| base64_decode(b.trim())) .and_then(|raw| String::from_utf8(raw).ok()) @@ -48,6 +48,16 @@ impl Auth { } } + /// The secret a SigV4 access key id `id` signs with, when it is not `id` itself: in Basic + /// mode, the password when `id` is the user (compared in constant time), so the secret + /// never crosses the wire. `None` otherwise, a bearer included: its id is its secret. + pub(crate) fn secret_for(&self, id: &str) -> Option<&str> { + match self { + Auth::Basic { user, pass } if constant_eq(id.as_bytes(), user.as_bytes()) => Some(pass), + _ => None, + } + } + /// The 401 challenge to return for an unauthenticated protected request. /// /// Each mode names its own scheme, unlike accounts mode, which challenges with `Basic` @@ -73,8 +83,8 @@ impl Auth { pub(crate) const BASIC_CHALLENGE: &str = "Basic realm=\"vk-registry\""; const BEARER_CHALLENGE: &str = "Bearer realm=\"vk-registry\""; -fn authorization(req: &Request) -> Option<&str> { - req.headers() +fn authorization(headers: &HeaderMap) -> Option<&str> { + headers .get(hyper::header::AUTHORIZATION) .and_then(|v| v.to_str().ok()) } @@ -133,6 +143,20 @@ mod tests { assert!(base64_decode("not base64!").is_none()); } + #[test] + fn only_the_basic_user_has_a_secret_other_than_itself() { + let basic = Auth::Basic { + user: "ci".into(), + pass: "pa55".into(), + }; + assert_eq!(basic.secret_for("ci"), Some("pa55")); + assert_eq!(basic.secret_for("pa55"), None); + assert_eq!(basic.secret_for("c"), None); + let bearer = Auth::Bearer { token: "t".into() }; + assert_eq!(bearer.secret_for("t"), None); + assert_eq!(Auth::None.secret_for("x"), None); + } + #[test] fn constant_eq_matches_only_equal() { assert!(constant_eq(b"secret", b"secret")); diff --git a/vk-registry/src/dav/mod.rs b/vk-registry/src/dav/mod.rs index b22e0be0..21102d3d 100644 --- a/vk-registry/src/dav/mod.rs +++ b/vk-registry/src/dav/mod.rs @@ -175,7 +175,7 @@ pub(crate) fn parse(rest: &str) -> Option<(Vec, bool)> { /// Percent-decode one path component: `%XX` escapes only, `+` being itself in a path, and /// the decoded bytes must be UTF-8. `None` for a malformed escape or invalid UTF-8. -fn decode_segment(raw: &str) -> Option { +pub(crate) fn decode_segment(raw: &str) -> Option { fn hex(c: u8) -> Option { match c { b'0'..=b'9' => Some(c - b'0'), diff --git a/vk-registry/src/files.rs b/vk-registry/src/files.rs index 5f885837..8fd1af22 100644 --- a/vk-registry/src/files.rs +++ b/vk-registry/src/files.rs @@ -352,6 +352,20 @@ where Ok((total, crate::hex_of(&hasher.finalize()))) } +/// Read and discard `body` up to `cap` bytes, and the frame that crosses it. For a refusal +/// answered before the caller is known, which must not make the server read a whole +/// upload: a body that ends within them is answered rather than reset, and what is left of a +/// longer one closes the connection after the answer, as every refusal of a request with a +/// body does. +pub(crate) async fn drain_up_to(body: B, cap: u64) +where + B: hyper::body::Body + Unpin, + B::Error: std::error::Error + Send + Sync + 'static, +{ + // Past `cap`, or unreadable: either way there is nothing more to read. + let _ = drain_into(&mut tokio::io::sink(), body, cap).await; +} + #[cfg(test)] mod tests { use super::*; diff --git a/vk-registry/src/lib.rs b/vk-registry/src/lib.rs index b8946db9..98aed77e 100644 --- a/vk-registry/src/lib.rs +++ b/vk-registry/src/lib.rs @@ -61,6 +61,7 @@ pub(crate) mod keys; pub mod lock; pub mod oidc; pub mod relay; +pub(crate) mod s3; pub(crate) mod tags; pub(crate) mod upload; @@ -135,7 +136,7 @@ impl ServerState { /// lock service, sign-ins, keys and captions (the accounts db, not the store) do not touch /// it, and the batched exists probe is a read sent as a POST; a tag deleted from the settings /// pages is a write. -fn is_store_write(method: &Method, path: &str) -> bool { +pub(crate) fn is_store_write(method: &Method, path: &str) -> bool { match method.as_str() { "GET" | "HEAD" | "OPTIONS" | "PROPFIND" => false, _ if path.starts_with("/lock/") || path == EXISTS_PATH => false, @@ -4386,7 +4387,7 @@ fn closing_after_refusal(mut resp: Response, has_body: bool) -> Response bool { +pub(crate) fn cross_site(headers: &hyper::HeaderMap) -> bool { headers .get("sec-fetch-site") .and_then(|v| v.to_str().ok()) @@ -4423,6 +4424,14 @@ async fn route(req: Request, state: Arc) -> Result, state: Arc) -> Result { - if auth.enabled() && !auth.allows(&req) { + if auth.enabled() && !auth.allows(req.headers()) { return Ok(auth.challenge()); } None diff --git a/vk-registry/src/s3/mod.rs b/vk-registry/src/s3/mod.rs new file mode 100644 index 00000000..cf66de58 --- /dev/null +++ b/vk-registry/src/s3/mod.rs @@ -0,0 +1,443 @@ +//! S3 under `/s3/`, path-style: the gate S3 clients are answered by. A request is +//! authenticated by a SigV4 `Authorization` header whose access key id is a credential this +//! registry already takes (an API key, a hub token, the shared bearer, or the shared Basic +//! user) and whose signing secret is that same string (the Basic password for the user), so +//! nothing new is stored; or by the bearer, Basic or session credential the other routes +//! take. Every answer carries an `x-amz-request-id`, and refusals are S3's XML `Error`. +//! Presigned URLs and temporary credentials are refused. + +mod sigv4; +mod xml; + +use anyhow::Result; +use hyper::body::Incoming; +use hyper::header::{ + AUTHORIZATION, DATE, HeaderMap, HeaderValue, RETRY_AFTER, X_CONTENT_TYPE_OPTIONS, +}; +use hyper::{Method, Request, Response, StatusCode}; +use vk_hub_proto::token::Claims; + +use crate::{Authenticator, Authz, Body, ServerState, accounts, hubtoken}; + +/// Whether `path` is one this module answers. +pub(crate) fn is_path(path: &str) -> bool { + path == "/s3" || path.starts_with("/s3/") +} + +/// How much of a request body is read before answering a refusal decided before the caller +/// is known: enough for a small write to see its answer rather than a reset, not so much +/// that anyone who can reach the port can make the server read a large upload for nothing. +const PRE_AUTH_DRAIN: u64 = 64 * 1024; + +/// The header naming each request, for a client's report to be matched to a server line. +const REQUEST_ID: &str = "x-amz-request-id"; + +/// Serve one `/s3/…` request, from its credentials on: [`crate::route`] hands every such +/// request here before its own gate when S3 is on. +pub(crate) async fn route(state: &ServerState, req: Request) -> Result> { + let id = accounts::random_token(8); + let path = req.uri().path().to_string(); + let ctx = Ctx { + id: &id, + path: &path, + }; + let mut resp = match serve(state, req, &ctx).await { + Ok(resp) => resp, + Err(e) => { + eprintln!("vk-registry: S3 request {id} failed: {e:#}"); + internal_error(&ctx) + } + }; + let headers = resp.headers_mut(); + headers.insert(REQUEST_ID, HeaderValue::from_str(&id)?); + headers.insert(X_CONTENT_TYPE_OPTIONS, HeaderValue::from_static("nosniff")); + Ok(resp) +} + +/// What every answer to a request names: its id and the path it asked for. +struct Ctx<'a> { + id: &'a str, + path: &'a str, +} + +impl Ctx<'_> { + fn error(&self, status: StatusCode, code: &str, message: &str) -> Response { + xml::error(status, code, message, &[], self.path, self.id) + } +} + +fn internal_error(ctx: &Ctx<'_>) -> Response { + ctx.error( + StatusCode::INTERNAL_SERVER_ERROR, + "InternalError", + "the server could not answer; its log names this request id", + ) +} + +/// Who a request was authenticated as: what [`Authz`] borrows from. +enum Who { + /// Shared-secret or open mode, where the gate is the whole authorization model. + NoScopes, + Principal(accounts::Principal), + Token(Claims), +} + +impl Who { + fn authz(&self) -> Authz<'_> { + match self { + Who::NoScopes => Authz::NoScopes, + Who::Principal(p) => Authz::Accounts(p), + Who::Token(c) => Authz::Token(c), + } + } +} + +async fn serve( + state: &ServerState, + req: Request, + ctx: &Ctx<'_>, +) -> Result> { + let headers = req.headers(); + // Browsers attach cookies or Basic credentials to cross-site requests, as on every + // other machine route. + if !matches!(req.method(), &Method::GET | &Method::HEAD) && crate::cross_site(headers) { + let resp = ctx.error( + StatusCode::FORBIDDEN, + "AccessDenied", + "a write to this registry cannot come from another site's page", + ); + return before_auth(req, resp).await; + } + // A credential in a URL ends up in logs, proxies' included. + if is_presigned(req.uri().query().unwrap_or("")) { + let resp = ctx.error( + StatusCode::BAD_REQUEST, + "AuthorizationQueryParametersError", + "presigned URLs are not accepted: sign the Authorization header", + ); + return before_auth(req, resp).await; + } + if headers.contains_key("x-amz-security-token") { + let resp = ctx.error( + StatusCode::BAD_REQUEST, + "InvalidToken", + "temporary credentials are not accepted: sign with a key of this registry", + ); + return before_auth(req, resp).await; + } + let signed = headers + .get(AUTHORIZATION) + .is_some_and(|v| v.as_bytes().starts_with(b"AWS4-HMAC-SHA256 ")); + let who = if signed { + signed_by(state, &req, ctx) + } else { + presented(state, &req, ctx) + }; + let who = match who { + Ok(who) => who, + Err(resp) => return before_auth(req, *resp).await, + }; + + // A hub managing the registry may make it refuse writes for a while. + if state.store.read_only() && crate::is_store_write(req.method(), ctx.path) { + let mut resp = ctx.error( + StatusCode::SERVICE_UNAVAILABLE, + "ServiceUnavailable", + "the registry is read-only for now", + ); + resp.headers_mut() + .insert(RETRY_AFTER, HeaderValue::from_static("60")); + return crate::dav::refuse(req, resp, ctx.path).await; + } + unserved(&who.authz(), req, ctx).await +} + +/// The answer to an authenticated request while no S3 operation is served: a read of an +/// object or a bucket finds nothing, whoever `_authz` names, and anything else is not +/// implemented. The body is read through first, as after any authenticated refusal. +async fn unserved( + _authz: &Authz<'_>, + req: Request, + ctx: &Ctx<'_>, +) -> Result> { + let rest = ctx.path.strip_prefix("/s3").unwrap_or_default(); + let rest = rest.strip_prefix('/').unwrap_or(rest); + let read = matches!(req.method(), &Method::GET | &Method::HEAD); + let resp = match rest.split_once('/') { + _ if !read || rest.is_empty() => ctx.error( + StatusCode::NOT_IMPLEMENTED, + "NotImplemented", + "this registry does not serve that operation", + ), + None | Some((_, "")) => ctx.error( + StatusCode::NOT_FOUND, + "NoSuchBucket", + "the specified bucket does not exist", + ), + Some(_) => ctx.error( + StatusCode::NOT_FOUND, + "NoSuchKey", + "the specified key does not exist", + ), + }; + crate::dav::refuse(req, resp, ctx.path).await +} + +/// Answer `resp` to a request refused before it was authenticated, having read at most +/// [`PRE_AUTH_DRAIN`] of its body (and the frame that crossed it). +async fn before_auth(req: Request, resp: Response) -> Result> { + crate::files::drain_up_to(req.into_body(), PRE_AUTH_DRAIN).await; + Ok(resp) +} + +/// Whether the query carries SigV4's query-string authentication. +fn is_presigned(query: &str) -> bool { + query.split('&').any(|pair| { + let name = pair.split_once('=').map_or(pair, |(n, _)| n); + matches!( + name, + "X-Amz-Algorithm" | "X-Amz-Credential" | "X-Amz-Signature" + ) + }) +} + +/// A request without a SigV4 header, authenticated as the other routes authenticate it: a +/// hub token, then the mode's own bearer, Basic or session credential. Refused with S3's 403 +/// rather than the 401 challenge, which an S3 client does not answer. +fn presented( + state: &ServerState, + req: &Request, + ctx: &Ctx<'_>, +) -> std::result::Result>> { + let denied = || { + let message = if req.headers().contains_key(AUTHORIZATION) { + "the credential was refused" + } else { + "this registry needs a credential: sign the request with a key of this registry" + }; + Box::new(ctx.error(StatusCode::FORBIDDEN, "AccessDenied", message)) + }; + if let Some(claims) = token(state, req.headers(), denied)? { + return Ok(Who::Token(claims)); + } + match &state.auth { + Authenticator::Accounts { db, .. } => { + match accounts::resolve_principal(db, req, state.cookies_are_secure()) { + Ok(Some(p)) => Ok(Who::Principal(p)), + Ok(None) => Err(denied()), + Err(e) => Err(resolve_failed(e, ctx)), + } + } + Authenticator::Shared(auth) if auth.allows(req.headers()) => Ok(Who::NoScopes), + Authenticator::Shared(_) => Err(denied()), + } +} + +/// The hub token `headers` present, checked: `None` when they present none or tokens are +/// not taken (the mode then decides), `refused` when the token is not good. +fn token( + state: &ServerState, + headers: &HeaderMap, + refused: impl FnOnce() -> Box>, +) -> std::result::Result, Box>> { + let Some(t) = hubtoken::presented(headers) else { + return Ok(None); + }; + match state.tokens.check(&t, vk_peer::now_secs()) { + Some(Ok(claims)) => Ok(Some(claims)), + Some(Err(why)) => { + state.tokens.log_refusal(&why); + Err(refused()) + } + None => Ok(None), + } +} + +/// The answer when the accounts database could not be read: the release taking over is +/// asked again, anything else is logged under the request's id. +fn resolve_failed(e: anyhow::Error, ctx: &Ctx<'_>) -> Box> { + if accounts::is_handed_over_error(&e) { + return Box::new(crate::ask_again(ctx.error( + StatusCode::SERVICE_UNAVAILABLE, + "ServiceUnavailable", + "this registry is updating; retry", + ))); + } + eprintln!( + "vk-registry: S3 request {}: resolving a credential: {e:#}", + ctx.id + ); + Box::new(internal_error(ctx)) +} + +/// A request with a SigV4 header. In order: the header parses and names S3's scope; the +/// request's time is on the scope's day and within [`sigv4::MAX_SKEW_SECS`] of the server's; +/// the access key id is authenticated exactly as a bearer would be, by the resolvers the +/// other routes use; only then is the secret it maps to used as a key, to check the +/// signature. On a server with no credential configured, an id that is not a hub token is +/// taken with its signature unchecked, as nothing else is checked there. +fn signed_by( + state: &ServerState, + req: &Request, + ctx: &Ctx<'_>, +) -> std::result::Result>> { + let headers = req.headers(); + let malformed = |why: &str| { + Box::new(ctx.error(StatusCode::BAD_REQUEST, "AuthorizationHeaderMalformed", why)) + }; + let value = headers + .get(AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .ok_or_else(|| malformed("the Authorization header is not text"))?; + let credential = sigv4::parse_authorization(value).map_err(malformed)?; + if let Some(why) = sigv4::s3_refusal(&credential) { + return Err(malformed(why)); + } + + let now = vk_peer::now_secs(); + let stamp = headers + .get("x-amz-date") + .or_else(|| headers.get(DATE)) + .and_then(|v| v.to_str().ok()) + .unwrap_or_default(); + match sigv4::check_clock(stamp, credential.date, now) { + Ok(()) => {} + Err(sigv4::Clock::Unreadable) => { + return Err(Box::new(ctx.error( + StatusCode::FORBIDDEN, + "AccessDenied", + "a signed request needs an x-amz-date header as YYYYMMDDTHHMMSSZ", + ))); + } + Err(sigv4::Clock::OtherDay) => { + return Err(malformed( + "the credential's date is not the day of x-amz-date", + )); + } + Err(sigv4::Clock::Skewed) => { + let server_time = sigv4::format_time(now); + let skew = (sigv4::MAX_SKEW_SECS * 1000).to_string(); + return Err(Box::new(xml::error( + StatusCode::FORBIDDEN, + "RequestTimeTooSkewed", + "the difference between the request time and the server's time is too large", + &[ + ("RequestTime", stamp), + ("ServerTime", &server_time), + ("MaxAllowedSkewMilliseconds", &skew), + ], + ctx.path, + ctx.id, + ))); + } + } + + // The id, authenticated as the bearer it would be: the one header map built here holds + // nothing else, so no resolver can accept through it what it would not accept as sent. + let unknown = || { + Box::new(ctx.error( + StatusCode::FORBIDDEN, + "InvalidAccessKeyId", + "the access key id is not a credential of this registry", + )) + }; + let as_bearer = bearer_of(credential.id).ok_or_else(unknown)?; + let (who, secret) = match (token(state, &as_bearer, unknown)?, &state.auth) { + (Some(claims), _) => (Who::Token(claims), Some(credential.id)), + (None, Authenticator::Accounts { db, .. }) => { + match accounts::resolve_headers(db, &as_bearer, state.cookies_are_secure()) { + Ok(Some(p)) => (Who::Principal(p), Some(credential.id)), + Ok(None) => return Err(unknown()), + Err(e) => return Err(resolve_failed(e, ctx)), + } + } + (None, Authenticator::Shared(auth)) if !auth.enabled() => (Who::NoScopes, None), + (None, Authenticator::Shared(auth)) => match auth.secret_for(credential.id) { + Some(password) => (Who::NoScopes, Some(password)), + None if auth.allows(&as_bearer) => (Who::NoScopes, Some(credential.id)), + None => return Err(unknown()), + }, + }; + let Some(secret) = secret else { + return Ok(who); + }; + + let payload_hash = headers + .get("x-amz-content-sha256") + .and_then(|v| v.to_str().ok()) + .ok_or_else(|| malformed("the signed x-amz-content-sha256 header is missing"))?; + let canonical = sigv4::canonical_request( + req.method().as_str(), + req.uri().path(), + req.uri().query().unwrap_or(""), + headers, + &credential.signed_headers, + payload_hash, + ) + .map_err(|why| match why { + sigv4::Canon::Path | sigv4::Canon::Query => Box::new(ctx.error( + StatusCode::BAD_REQUEST, + "InvalidURI", + "the request's URI has a malformed escape or is not UTF-8", + )), + sigv4::Canon::Missing(name) => malformed(&format!("the signed header {name} is missing")), + sigv4::Canon::NotText(name) => malformed(&format!("the signed header {name} is not text")), + })?; + if !sigv4::signature_matches(secret, &credential, stamp, &canonical) { + let sts = sigv4::string_to_sign(stamp, &credential.scope(), &canonical); + return Err(Box::new(xml::error( + StatusCode::FORBIDDEN, + "SignatureDoesNotMatch", + "the request signature does not match the one computed with the key's secret", + &[("StringToSign", &sts), ("CanonicalRequest", &canonical)], + ctx.path, + ctx.id, + ))); + } + Ok(who) +} + +/// `Authorization: Bearer `, alone; `None` when `id` cannot be a header value. +fn bearer_of(id: &str) -> Option { + let mut headers = HeaderMap::new(); + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {id}")).ok()?, + ); + Some(headers) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_s3_paths_are_s3() { + for p in ["/s3", "/s3/", "/s3/b", "/s3/b/k"] { + assert!(is_path(p), "{p}"); + } + for p in ["/s3x", "/s", "/dav/s3", "/S3/"] { + assert!(!is_path(p), "{p}"); + } + } + + #[test] + fn a_presigned_query_is_recognised() { + assert!(is_presigned( + "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=00" + )); + assert!(is_presigned("list-type=2&X-Amz-Credential=a%2Fb")); + assert!(is_presigned("X-Amz-Signature")); + assert!(!is_presigned("list-type=2&prefix=X-Amz-Algorithm")); + assert!(!is_presigned("x-amz-algorithm=1")); + assert!(!is_presigned("")); + } + + #[test] + fn the_id_becomes_one_bearer_header_or_nothing() { + let h = bearer_of("vkr_abc").unwrap(); + assert_eq!(h.len(), 1); + assert_eq!(h[AUTHORIZATION], "Bearer vkr_abc"); + assert!(bearer_of("a\nb").is_none()); + } +} diff --git a/vk-registry/src/s3/sigv4.rs b/vk-registry/src/s3/sigv4.rs new file mode 100644 index 00000000..80b6cb8b --- /dev/null +++ b/vk-registry/src/s3/sigv4.rs @@ -0,0 +1,879 @@ +//! AWS Signature Version 4 over the `Authorization` header: parsing the header, the clock +//! check, the canonical request, the string to sign and the signature. Pure functions: the +//! region, the service and the payload hash are parameters, so AWS's generic signing suite +//! (service `service`, no `x-amz-content-sha256`) drives them as it is; what S3 requires of a +//! request on top of that is [`s3_refusal`], which the gate applies. +//! +//! The canonical request is S3's: the path is never normalised (`.`, `..` and `//` are part +//! of a key), each segment is percent-decoded once and re-encoded with the unreserved set. + +use hmac::{Hmac, KeyInit, Mac}; +use hyper::HeaderMap; +use sha2::{Digest, Sha256}; +use zeroize::Zeroizing; + +/// The one algorithm taken, which is also the scheme of the `Authorization` header. +pub(crate) const ALGORITHM: &str = "AWS4-HMAC-SHA256"; + +/// How far a request's time may be from the server's, either way: AWS's window. +pub(crate) const MAX_SKEW_SECS: u64 = 15 * 60; + +/// The scope's last field, and the last HMAC of the signing key's chain. +const TERMINATOR: &str = "aws4_request"; + +/// A parsed `Authorization: AWS4-HMAC-SHA256 …` header, borrowing from its value. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Credential<'a> { + /// The access key id: everything before the scope's four fields, `/` included. + pub(crate) id: &'a str, + /// The scope's date, `YYYYMMDD`. + pub(crate) date: &'a str, + pub(crate) region: &'a str, + pub(crate) service: &'a str, + pub(crate) terminator: &'a str, + /// The signed header names, lowercase, sorted and without duplicates. + pub(crate) signed_headers: Vec<&'a str>, + /// The presented signature, 64 lowercase hex digits. + pub(crate) signature: &'a str, +} + +impl Credential<'_> { + /// `date/region/service/terminator`, as the string to sign carries it. + pub(crate) fn scope(&self) -> String { + format!( + "{}/{}/{}/{}", + self.date, self.region, self.service, self.terminator + ) + } +} + +/// Parse an `Authorization` value: `AWS4-HMAC-SHA256 Credential=//// +/// , SignedHeaders=, Signature=`, the fields in any order, separated +/// by commas with or without spaces. The scope is split from the right, so an id may hold a +/// `/` and a scope field may not. The reason a value is malformed otherwise. +pub(crate) fn parse_authorization(value: &str) -> Result, &'static str> { + let fields = value + .strip_prefix(ALGORITHM) + .and_then(|r| r.strip_prefix(' ')) + .ok_or("the algorithm must be AWS4-HMAC-SHA256")?; + let (mut credential, mut signed, mut signature) = (None, None, None); + for field in fields.split(',') { + let (name, v) = field + .trim() + .split_once('=') + .ok_or("each field must be Name=value")?; + let slot = match name { + "Credential" => &mut credential, + "SignedHeaders" => &mut signed, + "Signature" => &mut signature, + _ => return Err("unknown field: only Credential, SignedHeaders and Signature"), + }; + if slot.replace(v).is_some() { + return Err("a field is given twice"); + } + } + let credential = credential.ok_or("Credential is missing")?; + let signed = signed.ok_or("SignedHeaders is missing")?; + let signature = signature.ok_or("Signature is missing")?; + + let mut scope = credential.rsplitn(5, '/'); + let (Some(terminator), Some(service), Some(region), Some(date), Some(id)) = ( + scope.next(), + scope.next(), + scope.next(), + scope.next(), + scope.next(), + ) else { + return Err("Credential must be ////aws4_request"); + }; + if id.is_empty() || region.is_empty() || service.is_empty() || terminator.is_empty() { + return Err("Credential has an empty field"); + } + if date.len() != 8 || !date.bytes().all(|b| b.is_ascii_digit()) { + return Err("the credential's date must be YYYYMMDD"); + } + let mut signed_headers = Vec::new(); + for name in signed.split(';') { + if name.is_empty() || !name.bytes().all(is_lowercase_token) { + return Err("SignedHeaders must be lowercase header names separated by ';'"); + } + signed_headers.push(name); + } + signed_headers.sort_unstable(); + signed_headers.dedup(); + if signature.len() != 64 + || !signature + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + return Err("Signature must be 64 lowercase hex digits"); + } + Ok(Credential { + id, + date, + region, + service, + terminator, + signed_headers, + signature, + }) +} + +/// A header-name byte (RFC 9110 `tchar`), lowercase. +fn is_lowercase_token(b: u8) -> bool { + b.is_ascii_lowercase() || b.is_ascii_digit() || b"!#$%&'*+-.^_`|~".contains(&b) +} + +/// What S3 requires of a SigV4 request beyond the signature: its scope names the service +/// `s3` and ends in `aws4_request`, and it signs `host` (what it was sent to) and +/// `x-amz-content-sha256` (what it carries). The reason, when it does not. +pub(crate) fn s3_refusal(credential: &Credential<'_>) -> Option<&'static str> { + if credential.service != "s3" { + return Some("the credential's service must be s3"); + } + if credential.terminator != TERMINATOR { + return Some("the credential must end in aws4_request"); + } + for required in ["host", "x-amz-content-sha256"] { + if !credential.signed_headers.contains(&required) { + return Some(if required == "host" { + "SignedHeaders must include host" + } else { + "SignedHeaders must include x-amz-content-sha256" + }); + } + } + None +} + +/// Why a request's time is refused. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Clock { + /// Not ISO 8601 basic (`YYYYMMDDTHHMMSSZ`), or before 1970. + Unreadable, + /// Not on the credential scope's date. + OtherDay, + /// More than [`MAX_SKEW_SECS`] from `now`. + Skewed, +} + +/// Check the request's time `stamp` against the scope's `date` and the server's `now`. +pub(crate) fn check_clock(stamp: &str, date: &str, now: u64) -> Result<(), Clock> { + let at = parse_stamp(stamp).ok_or(Clock::Unreadable)?; + if stamp.get(..8) != Some(date) { + return Err(Clock::OtherDay); + } + if at.abs_diff(now) > MAX_SKEW_SECS { + return Err(Clock::Skewed); + } + Ok(()) +} + +/// Seconds since the epoch of an ISO 8601 basic UTC time, `YYYYMMDDTHHMMSSZ`. +pub(crate) fn parse_stamp(stamp: &str) -> Option { + let b = stamp.as_bytes(); + if b.len() != 16 || b.get(8) != Some(&b'T') || b.get(15) != Some(&b'Z') { + return None; + } + let num = |from: usize, to: usize| -> Option { + let digits = stamp.get(from..to)?; + digits + .bytes() + .all(|c| c.is_ascii_digit()) + .then(|| digits.parse().ok())? + }; + let (y, mo, d) = (num(0, 4)?, num(4, 6)?, num(6, 8)?); + let (h, mi, s) = (num(9, 11)?, num(11, 13)?, num(13, 15)?); + if !(1..=12).contains(&mo) || d == 0 || d > days_in_month(y, mo) || h > 23 || mi > 59 || s > 59 + { + return None; + } + let days = u64::try_from(days_from_civil(i64::from(y), mo, d)).ok()?; + Some(days * 86_400 + u64::from(h) * 3600 + u64::from(mi) * 60 + u64::from(s)) +} + +/// `secs` since the epoch as ISO 8601 extended UTC, `YYYY-MM-DDTHH:MM:SSZ`: how S3 writes +/// its `ServerTime`. +pub(crate) fn format_time(secs: u64) -> String { + // `u64::MAX / 86_400` fits an `i64`: this never falls back. + let days = i64::try_from(secs / 86_400).unwrap_or(0); + let (y, m, d) = civil_from_days(days); + let rem = secs % 86_400; + format!( + "{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}Z", + rem / 3600, + rem / 60 % 60, + rem % 60 + ) +} + +fn days_in_month(y: u32, m: u32) -> u32 { + match m { + 2 if y.is_multiple_of(4) && (!y.is_multiple_of(100) || y.is_multiple_of(400)) => 29, + 2 => 28, + 4 | 6 | 9 | 11 => 30, + _ => 31, + } +} + +/// Days since 1970-01-01 of a proleptic Gregorian date (H. Hinnant's `days_from_civil`). +fn days_from_civil(y: i64, m: u32, d: u32) -> i64 { + let y = if m <= 2 { y - 1 } else { y }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let (m, d) = (i64::from(m), i64::from(d)); + let doy = (153 * (if m > 2 { m - 3 } else { m + 9 }) + 2) / 5 + d - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + era * 146_097 + doe - 719_468 +} + +/// The date `days` after 1970-01-01 (H. Hinnant's `civil_from_days`). +fn civil_from_days(days: i64) -> (i64, i64, i64) { + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = yoe + era * 400 + i64::from(m <= 2); + (y, m, d) +} + +/// Why a request has no canonical form. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Canon<'a> { + /// A path segment with a malformed escape, or not UTF-8 once decoded. + Path, + /// The same, in a query name or value. + Query, + /// A signed header the request does not carry. + Missing(&'a str), + /// A signed header whose value is not text. + NotText(&'a str), +} + +/// The canonical request: method, canonical URI, canonical query, canonical headers, signed +/// header list and payload hash, one per line. `raw_path` and `raw_query` are as the request +/// line carries them; `payload_hash` is what the string to sign binds the body by (for S3, +/// the `x-amz-content-sha256` value as sent). +pub(crate) fn canonical_request<'s>( + method: &str, + raw_path: &str, + raw_query: &str, + headers: &HeaderMap, + signed: &[&'s str], + payload_hash: &str, +) -> Result> { + let uri = canonical_uri(raw_path).ok_or(Canon::Path)?; + let query = canonical_query(raw_query).ok_or(Canon::Query)?; + let mut out = format!("{method}\n{uri}\n{query}\n"); + for &name in signed { + let mut values = headers.get_all(name).iter().peekable(); + if values.peek().is_none() { + return Err(Canon::Missing(name)); + } + out.push_str(name); + out.push(':'); + for (i, v) in values.enumerate() { + let v = std::str::from_utf8(v.as_bytes()).map_err(|_| Canon::NotText(name))?; + if i > 0 { + out.push(','); + } + push_collapsed(&mut out, v); + } + out.push('\n'); + } + out.push('\n'); + out.push_str(&signed.join(";")); + out.push('\n'); + out.push_str(payload_hash); + Ok(out) +} + +/// `value` trimmed, each inner run of spaces and tabs written as one space. +fn push_collapsed(out: &mut String, value: &str) { + for (i, word) in value + .split([' ', '\t']) + .filter(|w| !w.is_empty()) + .enumerate() + { + if i > 0 { + out.push(' '); + } + out.push_str(word); + } +} + +/// The path, each `/`-separated segment percent-decoded then re-encoded, never normalised. +/// Splitting before decoding keeps an encoded `%2F` inside its segment. +pub(crate) fn canonical_uri(raw_path: &str) -> Option { + let mut out = String::with_capacity(raw_path.len()); + for (i, segment) in raw_path.split('/').enumerate() { + if i > 0 { + out.push('/'); + } + out.push_str(&reencode(segment)?); + } + Some(out) +} + +/// The query's pairs, each name and value decoded then re-encoded, a bare name as `name=`, +/// sorted by name then value. +pub(crate) fn canonical_query(raw_query: &str) -> Option { + let mut pairs = Vec::new(); + for pair in raw_query.split('&').filter(|p| !p.is_empty()) { + let (name, value) = pair.split_once('=').unwrap_or((pair, "")); + pairs.push((reencode(name)?, reencode(value)?)); + } + pairs.sort_unstable(); + let mut out = String::with_capacity(raw_query.len()); + for (i, (name, value)) in pairs.iter().enumerate() { + if i > 0 { + out.push('&'); + } + out.push_str(name); + out.push('='); + out.push_str(value); + } + Some(out) +} + +/// One URI component in SigV4's encoding: `%XX` escapes decoded (a `+` is itself), the +/// result encoded again with everything but the unreserved set escaped. +fn reencode(raw: &str) -> Option { + Some(crate::percent_encode(&crate::dav::decode_segment(raw)?)) +} + +/// The string to sign for a request made at `stamp` under `scope`. +pub(crate) fn string_to_sign(stamp: &str, scope: &str, canonical_request: &str) -> String { + let hashed = crate::hex_of(&Sha256::digest(canonical_request.as_bytes())); + format!("{ALGORITHM}\n{stamp}\n{scope}\n{hashed}") +} + +/// HMAC-SHA256 of `data` under `key`, in a buffer wiped on drop. `None` never happens for +/// HMAC, which takes a key of any length; it is not turned into a key of zeros instead. +fn hmac(key: &[u8], data: &[u8]) -> Option> { + let mut mac = as KeyInit>::new_from_slice(key).ok()?; + mac.update(data); + let mut out = Zeroizing::new([0u8; 32]); + out.copy_from_slice(&mac.finalize().into_bytes()); + Some(out) +} + +/// The hex signature of `string_to_sign` under the key derived from `secret` for the scope +/// `date/region/service/aws4_request`. The derived keys live in this call only, each wiped +/// when dropped; nothing but the signature leaves it. +pub(crate) fn signature( + secret: &str, + date: &str, + region: &str, + service: &str, + string_to_sign: &str, +) -> Option { + let seed = Zeroizing::new(format!("AWS4{secret}").into_bytes()); + let k_date = hmac(&seed, date.as_bytes())?; + let k_region = hmac(&*k_date, region.as_bytes())?; + let k_service = hmac(&*k_region, service.as_bytes())?; + let k_signing = hmac(&*k_service, TERMINATOR.as_bytes())?; + let signed = hmac(&*k_signing, string_to_sign.as_bytes())?; + Some(crate::hex_of(&*signed)) +} + +/// Whether `credential`'s signature is the one `secret` gives `canonical_request` made at +/// `stamp`, compared in constant time. +pub(crate) fn signature_matches( + secret: &str, + credential: &Credential<'_>, + stamp: &str, + canonical_request: &str, +) -> bool { + let sts = string_to_sign(stamp, &credential.scope(), canonical_request); + signature( + secret, + credential.date, + credential.region, + credential.service, + &sts, + ) + .is_some_and(|ours| crate::auth::constant_eq(ours.as_bytes(), credential.signature.as_bytes())) +} + +#[cfg(test)] +mod tests { + use super::*; + use hyper::header::{HeaderName, HeaderValue}; + + const EMPTY_SHA256: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + /// A request as the suite writes one: the request line, `Name:value` headers (a line + /// starting with whitespace continues the one above), a blank line, the body. + struct Request { + method: String, + path: String, + query: String, + headers: HeaderMap, + body: String, + } + + fn parse_request(text: &str) -> Request { + let (head, body) = text.split_once("\n\n").unwrap_or((text, "")); + let mut lines = head.lines(); + let line = lines.next().unwrap(); + let (method, target) = line.split_once(' ').unwrap(); + let target = target.strip_suffix(" HTTP/1.1").unwrap(); + let (path, query) = target.split_once('?').unwrap_or((target, "")); + let mut fields: Vec<(String, String)> = Vec::new(); + for line in lines { + if line.starts_with([' ', '\t']) { + let last = fields.last_mut().unwrap(); + last.1.push(' '); + last.1.push_str(line.trim()); + } else { + // As an HTTP parser does, the value without its surrounding whitespace. + let (name, value) = line.split_once(':').unwrap(); + fields.push((name.to_string(), value.trim().to_string())); + } + } + let mut headers = HeaderMap::new(); + for (name, value) in fields { + headers.append( + HeaderName::from_bytes(name.as_bytes()).unwrap(), + HeaderValue::from_str(&value).unwrap(), + ); + } + Request { + method: method.to_string(), + path: path.to_string(), + query: query.to_string(), + headers, + body: body.to_string(), + } + } + + /// Everything a header-signed request yields: its canonical request, string to sign and + /// signature under `secret`, and whether the signature it carries verifies. + struct Signed { + canonical: String, + string_to_sign: String, + signature: String, + verifies: bool, + } + + fn sign(req: &Request, secret: &str, payload_hash: &str) -> Signed { + let auth = req.headers["authorization"].to_str().unwrap(); + let cred = parse_authorization(auth).unwrap(); + let stamp = req.headers["x-amz-date"].to_str().unwrap(); + let canonical = canonical_request( + &req.method, + &req.path, + &req.query, + &req.headers, + &cred.signed_headers, + payload_hash, + ) + .unwrap(); + let string_to_sign = string_to_sign(stamp, &cred.scope(), &canonical); + let signature = signature( + secret, + cred.date, + cred.region, + cred.service, + &string_to_sign, + ) + .unwrap(); + let verifies = signature_matches(secret, &cred, stamp, &canonical); + Signed { + canonical, + string_to_sign, + signature, + verifies, + } + } + + /// Every case of AWS's signing suite (see `testdata/sigv4/README`) reproduces its + /// canonical request, string to sign and signature, and its signed request verifies. A + /// `-normalized` case is the request its `-unnormalized` twin signs, with the path + /// normalised as services other than S3 do: S3 does not, so its canonical request must be + /// the twin's. + #[test] + fn the_aws_signing_suite_verifies() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src/s3/testdata/sigv4"); + let mut cases: Vec<_> = std::fs::read_dir(&root) + .unwrap() + .map(|e| e.unwrap().path()) + .filter(|p| p.is_dir()) + .collect(); + cases.sort(); + assert_eq!(cases.len(), 38, "the vendored suite is whole"); + let read = |case: &std::path::Path, file: &str| { + std::fs::read_to_string(case.join(file)) + .unwrap_or_else(|e| panic!("{}/{file}: {e}", case.display())) + }; + let mut twinned = 0; + for case in &cases { + let name = case.file_name().unwrap().to_str().unwrap(); + let context: serde_json::Value = + serde_json::from_str(&read(case, "context.json")).unwrap(); + let secret = context["credentials"]["secret_access_key"] + .as_str() + .unwrap(); + let req = parse_request(&read(case, "header-signed-request.txt")); + let payload_hash = match req.headers.get("x-amz-content-sha256") { + Some(v) => v.to_str().unwrap().to_string(), + None if context["sign_body"] == true => { + crate::hex_of(&Sha256::digest(req.body.as_bytes())) + } + None => EMPTY_SHA256.to_string(), + }; + let signed = sign(&req, secret, &payload_hash); + if let Some(stem) = name.strip_suffix("-normalized") { + assert_eq!(context["normalize"], true, "{name}"); + let twin = root.join(format!("{stem}-unnormalized")); + assert_eq!( + signed.canonical, + read(&twin, "header-canonical-request.txt"), + "{name}: S3 signs the path as sent" + ); + twinned += 1; + continue; + } + assert_eq!( + signed.canonical, + read(case, "header-canonical-request.txt"), + "{name}" + ); + assert_eq!( + signed.string_to_sign, + read(case, "header-string-to-sign.txt"), + "{name}" + ); + assert_eq!( + signed.signature, + read(case, "header-signature.txt").trim(), + "{name}" + ); + assert!(signed.verifies, "{name}"); + } + assert_eq!(twinned, 7, "every -normalized case has its twin"); + } + + const S3_SECRET: &str = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + + /// AWS's S3 examples ("Signature Calculations for the Authorization Header: Transferring + /// Payload in a Single Chunk"): each request, as written there with its Authorization + /// header, parses, passes S3's policy, and verifies to the documented canonical request + /// and signature. Virtual-hosted, as AWS signs them, which changes nothing here: the + /// signature covers the `Host` and path as sent. + #[test] + fn the_aws_s3_examples_verify() { + let date = "x-amz-date:20130524T000000Z"; + let empty = format!("x-amz-content-sha256:{EMPTY_SHA256}"); + let put_hash = "44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072"; + let cases: [(&str, &str, Vec, &str, &str); 4] = [ + ( + "GET /test.txt", + "host;range;x-amz-content-sha256;x-amz-date", + vec!["range:bytes=0-9".into(), empty.clone(), date.into()], + "GET\n/test.txt\n\nhost:examplebucket.s3.amazonaws.com\nrange:bytes=0-9\n\ + x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n\ + x-amz-date:20130524T000000Z\n\nhost;range;x-amz-content-sha256;x-amz-date\n\ + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41", + ), + ( + "PUT /test$file.text", + "date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class", + vec![ + "date:Fri, 24 May 2013 00:00:00 GMT".into(), + date.into(), + "x-amz-storage-class:REDUCED_REDUNDANCY".into(), + format!("x-amz-content-sha256:{put_hash}"), + ], + "PUT\n/test%24file.text\n\ndate:Fri, 24 May 2013 00:00:00 GMT\n\ + host:examplebucket.s3.amazonaws.com\n\ + x-amz-content-sha256:44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072\n\ + x-amz-date:20130524T000000Z\nx-amz-storage-class:REDUCED_REDUNDANCY\n\n\ + date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class\n\ + 44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072", + "98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd", + ), + ( + "GET /?lifecycle", + "host;x-amz-content-sha256;x-amz-date", + vec![date.into(), empty.clone()], + "GET\n/\nlifecycle=\nhost:examplebucket.s3.amazonaws.com\n\ + x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n\ + x-amz-date:20130524T000000Z\n\nhost;x-amz-content-sha256;x-amz-date\n\ + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "fea454ca298b7da1c68078a5d1bdbfbbe0d65c699e0f91ac7a200a0136783543", + ), + ( + "GET /?max-keys=2&prefix=J", + "host;x-amz-content-sha256;x-amz-date", + vec![date.into(), empty.clone()], + "GET\n/\nmax-keys=2&prefix=J\nhost:examplebucket.s3.amazonaws.com\n\ + x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n\ + x-amz-date:20130524T000000Z\n\nhost;x-amz-content-sha256;x-amz-date\n\ + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7", + ), + ]; + for (line, signed, extra, canonical, sig) in cases { + let mut text = format!( + "{line} HTTP/1.1\nHost: examplebucket.s3.amazonaws.com\nAuthorization: \ + AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/\ + aws4_request,SignedHeaders={signed},Signature={sig}\n" + ); + for h in &extra { + text.push_str(h); + text.push('\n'); + } + let req = parse_request(&text); + let cred = parse_authorization(req.headers["authorization"].to_str().unwrap()).unwrap(); + assert_eq!(cred.id, "AKIAIOSFODNN7EXAMPLE"); + assert_eq!(s3_refusal(&cred), None, "{line}"); + let payload = req.headers["x-amz-content-sha256"].to_str().unwrap(); + let signed = sign(&req, S3_SECRET, payload); + assert_eq!(signed.canonical, canonical, "{line}"); + assert_eq!(signed.signature, sig, "{line}"); + assert!(signed.verifies, "{line}"); + // Another secret, or one byte of the request changed, and it no longer does. + assert!(!sign(&req, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEz", payload).verifies); + let moved = parse_request(&text.replacen(" /", " /x", 1)); + assert!(!sign(&moved, S3_SECRET, payload).verifies, "{line}"); + } + } + + fn header(pairs: &[(&str, &str)]) -> HeaderMap { + let mut h = HeaderMap::new(); + for (n, v) in pairs { + h.append( + HeaderName::from_bytes(n.as_bytes()).unwrap(), + HeaderValue::from_str(v).unwrap(), + ); + } + h + } + + #[test] + fn a_query_is_canonical_whatever_its_order_and_encoding() { + let want = "%E1%88%B4=v&a=1&a=2&b=x%20y&c=&d=%2B"; + for q in [ + "b=x%20y&a=2&a=1&c=&d=%2B&%E1%88%B4=v", + "a=1&%e1%88%b4=v&b=x%20y&a=2&c&d=%2b", + "d=%2B&c&&b=x%20y&a=%32&a=%31&ሴ=v", + ] { + assert_eq!(canonical_query(q).as_deref(), Some(want), "{q}"); + } + // A `+` is itself, encoded, not a space; a bare key and an empty value are the same. + assert_eq!(canonical_query("k=a+b").as_deref(), Some("k=a%2Bb")); + assert_eq!(canonical_query("uploads").as_deref(), Some("uploads=")); + assert_eq!(canonical_query("uploads=").as_deref(), Some("uploads=")); + assert_eq!(canonical_query("").as_deref(), Some("")); + // Sorted by name, then by value, on the encoded bytes. + assert_eq!( + canonical_query("x=b&x=a&X=c").as_deref(), + Some("X=c&x=a&x=b") + ); + assert_eq!(canonical_query("a=%zz"), None); + assert_eq!(canonical_query("a=%ff"), None, "not UTF-8 once decoded"); + } + + #[test] + fn a_path_is_encoded_once_and_never_normalised() { + for (raw, want) in [ + ("/", "/"), + ("/b/a%20b/c", "/b/a%20b/c"), + ("/b/a b", "/b/a%20b"), + ("/b/%7Ex~", "/b/~x~"), + ("/b/a%2Fb", "/b/a%2Fb"), + ("/b//./../k/", "/b//./../k/"), + ("/b/a+b$", "/b/a%2Bb%24"), + ] { + assert_eq!(canonical_uri(raw).as_deref(), Some(want), "{raw}"); + } + assert_eq!(canonical_uri("/b/%2"), None); + assert_eq!(canonical_uri("/b/%C3%28"), None); + } + + #[test] + fn header_values_are_trimmed_collapsed_and_joined() { + let h = header(&[ + ("host", "h"), + ("x-amz-meta-a", " one \t two three "), + ("x-amz-meta-b", "1"), + ("x-amz-meta-b", " 2 "), + ("x-amz-meta-b", "3"), + ("x-amz-meta-unsigned", "ignored"), + ]); + let creq = canonical_request( + "GET", + "/", + "", + &h, + &["host", "x-amz-meta-a", "x-amz-meta-b"], + "UNSIGNED-PAYLOAD", + ) + .unwrap(); + assert_eq!( + creq, + "GET\n/\n\nhost:h\nx-amz-meta-a:one two three\nx-amz-meta-b:1,2,3\n\n\ + host;x-amz-meta-a;x-amz-meta-b\nUNSIGNED-PAYLOAD" + ); + assert_eq!( + canonical_request("GET", "/", "", &h, &["host", "x-amz-date"], "-"), + Err(Canon::Missing("x-amz-date")) + ); + let mut binary = HeaderMap::new(); + binary.insert("x-b", HeaderValue::from_bytes(&[0xff, b'a']).unwrap()); + assert_eq!( + canonical_request("GET", "/", "", &binary, &["x-b"], "-"), + Err(Canon::NotText("x-b")) + ); + } + + const SIG: &str = "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"; + + fn authorization(credential: &str, signed: &str) -> String { + format!("AWS4-HMAC-SHA256 Credential={credential}, SignedHeaders={signed}, Signature={SIG}") + } + + #[test] + fn the_scope_is_split_from_the_right() { + let value = authorization( + "team/ci/key/20130524/auto/s3/aws4_request", + "x-amz-date;host;x-amz-content-sha256", + ); + let cred = parse_authorization(&value).unwrap(); + assert_eq!(cred.id, "team/ci/key"); + assert_eq!(cred.date, "20130524"); + assert_eq!(cred.region, "auto"); + assert_eq!(cred.service, "s3"); + assert_eq!(cred.terminator, "aws4_request"); + assert_eq!(cred.scope(), "20130524/auto/s3/aws4_request"); + assert_eq!( + cred.signed_headers, + ["host", "x-amz-content-sha256", "x-amz-date"], + "sorted" + ); + assert_eq!(cred.signature, SIG); + assert_eq!(s3_refusal(&cred), None); + } + + #[test] + fn a_malformed_authorization_is_refused() { + let ok = "id/20130524/us-east-1/s3/aws4_request"; + let signed = "host;x-amz-content-sha256"; + for bad in [ + format!("AWS4-HMAC-SHA1 Credential={ok}, SignedHeaders={signed}, Signature={SIG}"), + format!("AWS4-HMAC-SHA256 Credential={ok}, SignedHeaders={signed}"), + format!("AWS4-HMAC-SHA256 Credential={ok}, Signature={SIG}"), + format!("AWS4-HMAC-SHA256 SignedHeaders={signed}, Signature={SIG}"), + format!("{}, Signature={SIG}", authorization(ok, signed)), + format!("{}, Other=1", authorization(ok, signed)), + format!("{} junk", authorization(ok, signed)), + authorization("20130524/us-east-1/s3/aws4_request", signed), + authorization("/20130524/us-east-1/s3/aws4_request", signed), + authorization("id/2013052/us-east-1/s3/aws4_request", signed), + authorization("id/2013O524/us-east-1/s3/aws4_request", signed), + authorization("id/20130524//s3/aws4_request", signed), + authorization(ok, "Host;x-amz-content-sha256"), + authorization(ok, "host;;x-amz-content-sha256"), + authorization(ok, "host x-amz-content-sha256"), + authorization(ok, signed).replace(SIG, &SIG.to_uppercase()), + authorization(ok, signed).replace(SIG, &SIG[1..]), + String::new(), + "AWS4-HMAC-SHA256".to_string(), + "AWS4-HMAC-SHA256 ".to_string(), + ] { + assert!(parse_authorization(&bad).is_err(), "{bad}"); + } + } + + #[test] + fn s3_requires_its_service_and_both_headers_signed() { + let refusal = |credential: &str, signed: &str| { + s3_refusal(&parse_authorization(&authorization(credential, signed)).unwrap()) + }; + let ok = "id/20130524/us-east-1/s3/aws4_request"; + assert_eq!(refusal(ok, "host;x-amz-content-sha256;x-amz-date"), None); + assert!(refusal(ok, "x-amz-content-sha256;x-amz-date").is_some()); + assert!(refusal(ok, "host;x-amz-date").is_some()); + assert!( + refusal( + "id/20130524/us-east-1/service/aws4_request", + "host;x-amz-content-sha256" + ) + .is_some() + ); + assert!( + refusal( + "id/20130524/us-east-1/s3/aws4_requesT", + "host;x-amz-content-sha256" + ) + .is_some() + ); + } + + #[test] + fn the_clock_must_be_on_the_scope_day_and_within_fifteen_minutes() { + let at = parse_stamp("20130524T120000Z").unwrap(); + assert_eq!(at, 1_369_396_800); + assert_eq!(format_time(at), "2013-05-24T12:00:00Z"); + assert_eq!(check_clock("20130524T120000Z", "20130524", at), Ok(())); + for ok in [14 * 60 + 59, 15 * 60] { + assert_eq!(check_clock("20130524T120000Z", "20130524", at + ok), Ok(())); + assert_eq!(check_clock("20130524T120000Z", "20130524", at - ok), Ok(())); + } + assert_eq!( + check_clock("20130524T120000Z", "20130524", at + 15 * 60 + 1), + Err(Clock::Skewed) + ); + assert_eq!( + check_clock("20130524T120000Z", "20130524", at - 15 * 60 - 1), + Err(Clock::Skewed) + ); + // On the day after the scope's, even within the window. + assert_eq!( + check_clock("20130525T000100Z", "20130524", at + 12 * 3600), + Err(Clock::OtherDay) + ); + for bad in [ + "", + "20130524T120000", + "20130524 120000Z", + "2013-05-24T12:00:00Z", + "20131324T120000Z", + "20130230T120000Z", + "20130524T240000Z", + "20130524T126000Z", + "+0130524T120000Z", + "Fri, 24 May 2013 12:00:00 GMT", + ] { + assert_eq!( + check_clock(bad, "20130524", at), + Err(Clock::Unreadable), + "{bad}" + ); + } + assert_eq!(parse_stamp("19700101T000000Z"), Some(0)); + assert_eq!(parse_stamp("20000229T235959Z"), Some(951_868_799)); + assert_eq!(format_time(951_868_799), "2000-02-29T23:59:59Z"); + } + + /// What signing hands back is the hex signature and nothing else: no derived key is + /// returned or kept, so none outlives the call. + #[test] + fn signing_returns_the_signature_only() { + let f: fn(&str, &str, &str, &str, &str) -> Option = signature; + let sig = f(S3_SECRET, "20130524", "us-east-1", "s3", "x").unwrap(); + assert_eq!(sig.len(), 64); + assert_ne!( + sig, + f(S3_SECRET, "20130525", "us-east-1", "s3", "x").unwrap() + ); + assert_ne!(sig, f(S3_SECRET, "20130524", "auto", "s3", "x").unwrap()); + } +} diff --git a/vk-registry/src/s3/testdata/sigv4/LICENSE b/vk-registry/src/s3/testdata/sigv4/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vk-registry/src/s3/testdata/sigv4/README b/vk-registry/src/s3/testdata/sigv4/README new file mode 100644 index 00000000..468e6028 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/README @@ -0,0 +1,14 @@ +AWS Signature Version 4 signing test suite, header-signing cases + +Source: awslabs/aws-c-auth, tests/aws-signing-test-suite/v4 +https://github.com/awslabs/aws-c-auth/tree/3281f8692e6fd10562c4585a4dded5c16b322698/tests/aws-signing-test-suite/v4 +(the last commit touching that directory). Licensed under the Apache License 2.0; see +LICENSE beside this file. + +Each directory is one case, copied unchanged: context.json (credentials, region, service, +timestamp, whether the path is normalised and the body signed), request.txt (the request +before signing), header-signed-request.txt (the same, signed in the Authorization header), +header-canonical-request.txt, header-string-to-sign.txt and header-signature.txt. The +query-* files (presigned URLs, which vk-registry refuses) are left out. + +s3::sigv4's tests read them from here (the_aws_signing_suite_verifies). diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/context.json b/vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/header-canonical-request.txt new file mode 100644 index 00000000..fa8f49a1 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/header-canonical-request.txt @@ -0,0 +1,9 @@ +GET +/ + +host:example.amazonaws.com +my-header1:value2,value2,value1 +x-amz-date:20150830T123600Z + +host;my-header1;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/header-signature.txt new file mode 100644 index 00000000..7fc5d7ac --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/header-signature.txt @@ -0,0 +1 @@ +c9d5ea9f3f72853aea855b47ea873832890dbdd183b4468f858259531a5138ea \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/header-signed-request.txt new file mode 100644 index 00000000..36a12051 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/header-signed-request.txt @@ -0,0 +1,8 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:value2 +My-Header1:value2 +My-Header1:value1 +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=c9d5ea9f3f72853aea855b47ea873832890dbdd183b4468f858259531a5138ea + diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/header-string-to-sign.txt new file mode 100644 index 00000000..48a135ec --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +dc7f04a3abfde8d472b0ab1a418b741b7c67174dad1551b4117b15527fbe966c \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/request.txt b/vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/request.txt new file mode 100644 index 00000000..40455cec --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-key-duplicate/request.txt @@ -0,0 +1,5 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:value2 +My-Header1:value2 +My-Header1:value1 diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/context.json b/vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/header-canonical-request.txt new file mode 100644 index 00000000..8cb54769 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/header-canonical-request.txt @@ -0,0 +1,9 @@ +GET +/ + +host:example.amazonaws.com +my-header1:value1 value2 value3 +x-amz-date:20150830T123600Z + +host;my-header1;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/header-signature.txt new file mode 100644 index 00000000..e6ff7a91 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/header-signature.txt @@ -0,0 +1 @@ +cfd34249e4b1c8d6b91ef74165d41a32e5fab3306300901bb65a51a73575eefd \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/header-signed-request.txt new file mode 100644 index 00000000..b5ef6a00 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/header-signed-request.txt @@ -0,0 +1,8 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:value1 + value2 + value3 +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=cfd34249e4b1c8d6b91ef74165d41a32e5fab3306300901bb65a51a73575eefd + diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/header-string-to-sign.txt new file mode 100644 index 00000000..97c74309 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +e99419459a677bc11de234014be3c4e72c1ea5b454ceb58b613061f5d7a162e8 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/request.txt b/vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/request.txt new file mode 100644 index 00000000..b5026cd0 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-value-multiline/request.txt @@ -0,0 +1,5 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:value1 + value2 + value3 diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-value-order/context.json b/vk-registry/src/s3/testdata/sigv4/get-header-value-order/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-value-order/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-value-order/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-header-value-order/header-canonical-request.txt new file mode 100644 index 00000000..e336bc94 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-value-order/header-canonical-request.txt @@ -0,0 +1,9 @@ +GET +/ + +host:example.amazonaws.com +my-header1:value4,value1,value3,value2 +x-amz-date:20150830T123600Z + +host;my-header1;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-value-order/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-header-value-order/header-signature.txt new file mode 100644 index 00000000..8ef04944 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-value-order/header-signature.txt @@ -0,0 +1 @@ +08c7e5a9acfcfeb3ab6b2185e75ce8b1deb5e634ec47601a50643f830c755c01 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-value-order/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-header-value-order/header-signed-request.txt new file mode 100644 index 00000000..b786dc92 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-value-order/header-signed-request.txt @@ -0,0 +1,9 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:value4 +My-Header1:value1 +My-Header1:value3 +My-Header1:value2 +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=08c7e5a9acfcfeb3ab6b2185e75ce8b1deb5e634ec47601a50643f830c755c01 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-value-order/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-header-value-order/header-string-to-sign.txt new file mode 100644 index 00000000..711a8d4d --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-value-order/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +31ce73cd3f3d9f66977ad3dd957dc47af14df92fcd8509f59b349e9137c58b86 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-value-order/request.txt b/vk-registry/src/s3/testdata/sigv4/get-header-value-order/request.txt new file mode 100644 index 00000000..2c6f49d3 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-value-order/request.txt @@ -0,0 +1,6 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:value4 +My-Header1:value1 +My-Header1:value3 +My-Header1:value2 diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-value-trim/context.json b/vk-registry/src/s3/testdata/sigv4/get-header-value-trim/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-value-trim/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-value-trim/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-header-value-trim/header-canonical-request.txt new file mode 100644 index 00000000..a59087c9 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-value-trim/header-canonical-request.txt @@ -0,0 +1,10 @@ +GET +/ + +host:example.amazonaws.com +my-header1:value1 +my-header2:"a b c" +x-amz-date:20150830T123600Z + +host;my-header1;my-header2;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-value-trim/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-header-value-trim/header-signature.txt new file mode 100644 index 00000000..36081a9d --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-value-trim/header-signature.txt @@ -0,0 +1 @@ +acc3ed3afb60bb290fc8d2dd0098b9911fcaa05412b367055dee359757a9c736 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-value-trim/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-header-value-trim/header-signed-request.txt new file mode 100644 index 00000000..6331d16d --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-value-trim/header-signed-request.txt @@ -0,0 +1,7 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +My-Header1: value1 +My-Header2: "a b c" +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;my-header2;x-amz-date, Signature=acc3ed3afb60bb290fc8d2dd0098b9911fcaa05412b367055dee359757a9c736 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-value-trim/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-header-value-trim/header-string-to-sign.txt new file mode 100644 index 00000000..a0b15cc7 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-value-trim/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +a726db9b0df21c14f559d0a978e563112acb1b9e05476f0a6a1c7d68f28605c7 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-header-value-trim/request.txt b/vk-registry/src/s3/testdata/sigv4/get-header-value-trim/request.txt new file mode 100644 index 00000000..ad1ea986 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-header-value-trim/request.txt @@ -0,0 +1,4 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +My-Header1: value1 +My-Header2: "a b c" diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-normalized/context.json b/vk-registry/src/s3/testdata/sigv4/get-relative-normalized/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-normalized/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-normalized/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-normalized/header-canonical-request.txt new file mode 100644 index 00000000..ed91561f --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-normalized/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-normalized/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-normalized/header-signature.txt new file mode 100644 index 00000000..87f295fb --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-normalized/header-signature.txt @@ -0,0 +1 @@ +5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-normalized/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-normalized/header-signed-request.txt new file mode 100644 index 00000000..0de83271 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-normalized/header-signed-request.txt @@ -0,0 +1,5 @@ +GET /example/.. HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-normalized/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-normalized/header-string-to-sign.txt new file mode 100644 index 00000000..b187649c --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-normalized/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-normalized/request.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-normalized/request.txt new file mode 100644 index 00000000..6aa9f995 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-normalized/request.txt @@ -0,0 +1,2 @@ +GET /example/.. HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/context.json b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/header-canonical-request.txt new file mode 100644 index 00000000..ed91561f --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/header-signature.txt new file mode 100644 index 00000000..87f295fb --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/header-signature.txt @@ -0,0 +1 @@ +5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/header-signed-request.txt new file mode 100644 index 00000000..f9b38158 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/header-signed-request.txt @@ -0,0 +1,5 @@ +GET /example1/example2/../.. HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/header-string-to-sign.txt new file mode 100644 index 00000000..b187649c --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/request.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/request.txt new file mode 100644 index 00000000..a856c107 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-normalized/request.txt @@ -0,0 +1,2 @@ +GET /example1/example2/../.. HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/context.json b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/context.json new file mode 100644 index 00000000..7bee5c78 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": false, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/header-canonical-request.txt new file mode 100644 index 00000000..6919c44e --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/example1/example2/../.. + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/header-signature.txt new file mode 100644 index 00000000..4092d83d --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/header-signature.txt @@ -0,0 +1 @@ +dc33e0856fd4baca4d7aa2146c38958283844764f38c74252a333df5e613003b \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/header-signed-request.txt new file mode 100644 index 00000000..e731b35d --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/header-signed-request.txt @@ -0,0 +1,5 @@ +GET /example1/example2/../.. HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=dc33e0856fd4baca4d7aa2146c38958283844764f38c74252a333df5e613003b + diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/header-string-to-sign.txt new file mode 100644 index 00000000..4f457522 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +76115030c0f3ff06c20fdff5ceb6d5e0b835a1743e00b94fea7c7f381269437b \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/request.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/request.txt new file mode 100644 index 00000000..a856c107 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-relative-unnormalized/request.txt @@ -0,0 +1,2 @@ +GET /example1/example2/../.. HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/context.json b/vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/context.json new file mode 100644 index 00000000..7bee5c78 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": false, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/header-canonical-request.txt new file mode 100644 index 00000000..f640bc2c --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/example/.. + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/header-signature.txt new file mode 100644 index 00000000..d957303b --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/header-signature.txt @@ -0,0 +1 @@ +eca7ead57bb5aa5c8e28007acd4ff04e1ff9a0ff3b237ec1554a184887ff9282 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/header-signed-request.txt new file mode 100644 index 00000000..4f42ffca --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/header-signed-request.txt @@ -0,0 +1,5 @@ +GET /example/.. HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=eca7ead57bb5aa5c8e28007acd4ff04e1ff9a0ff3b237ec1554a184887ff9282 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/header-string-to-sign.txt new file mode 100644 index 00000000..ca54bda2 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +0511f456aa502b456d135fcb9d749374a55228f9dbeedda1eacf659e05b0615b \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/request.txt b/vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/request.txt new file mode 100644 index 00000000..6aa9f995 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-relative-unnormalized/request.txt @@ -0,0 +1,2 @@ +GET /example/.. HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/context.json b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/header-canonical-request.txt new file mode 100644 index 00000000..ed91561f --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/header-signature.txt new file mode 100644 index 00000000..87f295fb --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/header-signature.txt @@ -0,0 +1 @@ +5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/header-signed-request.txt new file mode 100644 index 00000000..99c8eebb --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/header-signed-request.txt @@ -0,0 +1,5 @@ +GET /./ HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/header-string-to-sign.txt new file mode 100644 index 00000000..b187649c --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/request.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/request.txt new file mode 100644 index 00000000..2c5ff385 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-normalized/request.txt @@ -0,0 +1,2 @@ +GET /./ HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/context.json b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/context.json new file mode 100644 index 00000000..7bee5c78 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": false, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/header-canonical-request.txt new file mode 100644 index 00000000..51590482 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/./ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/header-signature.txt new file mode 100644 index 00000000..b49ff0ee --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/header-signature.txt @@ -0,0 +1 @@ +68714168e6557f8f2de0ef956fc24dc2593a4bd2961f8df51898d8a134695145 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/header-signed-request.txt new file mode 100644 index 00000000..aeb1ffa1 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/header-signed-request.txt @@ -0,0 +1,5 @@ +GET /./ HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=68714168e6557f8f2de0ef956fc24dc2593a4bd2961f8df51898d8a134695145 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/header-string-to-sign.txt new file mode 100644 index 00000000..b7d8caf2 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +d67825e2268bd77a97c7688b8d72c31a3c1855b309808505ba0a9747d2465aa7 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/request.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/request.txt new file mode 100644 index 00000000..2c5ff385 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-dot-slash-unnormalized/request.txt @@ -0,0 +1,2 @@ +GET /./ HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-normalized/context.json b/vk-registry/src/s3/testdata/sigv4/get-slash-normalized/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-normalized/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-normalized/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-normalized/header-canonical-request.txt new file mode 100644 index 00000000..ed91561f --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-normalized/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-normalized/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-normalized/header-signature.txt new file mode 100644 index 00000000..87f295fb --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-normalized/header-signature.txt @@ -0,0 +1 @@ +5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-normalized/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-normalized/header-signed-request.txt new file mode 100644 index 00000000..b91087b1 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-normalized/header-signed-request.txt @@ -0,0 +1,5 @@ +GET // HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-normalized/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-normalized/header-string-to-sign.txt new file mode 100644 index 00000000..b187649c --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-normalized/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-normalized/request.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-normalized/request.txt new file mode 100644 index 00000000..25bc21e3 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-normalized/request.txt @@ -0,0 +1,2 @@ +GET // HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/context.json b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/header-canonical-request.txt new file mode 100644 index 00000000..915c57f2 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/example + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/header-signature.txt new file mode 100644 index 00000000..8f892bfa --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/header-signature.txt @@ -0,0 +1 @@ +ef75d96142cf21edca26f06005da7988e4f8dc83a165a80865db7089db637ec5 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/header-signed-request.txt new file mode 100644 index 00000000..2a1895ca --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/header-signed-request.txt @@ -0,0 +1,5 @@ +GET /./example HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=ef75d96142cf21edca26f06005da7988e4f8dc83a165a80865db7089db637ec5 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/header-string-to-sign.txt new file mode 100644 index 00000000..7429923e --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +214d50c111a8edc4819da6a636336472c916b5240f51e9a51b5c3305180cf702 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/request.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/request.txt new file mode 100644 index 00000000..5d919c9b --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-normalized/request.txt @@ -0,0 +1,2 @@ +GET /./example HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/context.json b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/context.json new file mode 100644 index 00000000..7bee5c78 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": false, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/header-canonical-request.txt new file mode 100644 index 00000000..36698c70 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/./example + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/header-signature.txt new file mode 100644 index 00000000..4a4b8f24 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/header-signature.txt @@ -0,0 +1 @@ +beb03f223f7deae4146464f06e29eebbee9c8afbe15c290cf07aa8b119e14cff \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/header-signed-request.txt new file mode 100644 index 00000000..d462bcd6 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/header-signed-request.txt @@ -0,0 +1,5 @@ +GET /./example HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=beb03f223f7deae4146464f06e29eebbee9c8afbe15c290cf07aa8b119e14cff + diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/header-string-to-sign.txt new file mode 100644 index 00000000..8be78c9c --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +73895e0e829507e28e39fd24669aedc2434a8e179e547e3c075b42921f952cdb \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/request.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/request.txt new file mode 100644 index 00000000..5d919c9b --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-pointless-dot-unnormalized/request.txt @@ -0,0 +1,2 @@ +GET /./example HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/context.json b/vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/context.json new file mode 100644 index 00000000..7bee5c78 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": false, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/header-canonical-request.txt new file mode 100644 index 00000000..5860c8a7 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +// + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/header-signature.txt new file mode 100644 index 00000000..83104716 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/header-signature.txt @@ -0,0 +1 @@ +c88bcd3d312d75078c0cd961d6deae3f4c754924b01669efcfcb439fd5e5b76e \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/header-signed-request.txt new file mode 100644 index 00000000..2a7afef5 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/header-signed-request.txt @@ -0,0 +1,5 @@ +GET // HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=c88bcd3d312d75078c0cd961d6deae3f4c754924b01669efcfcb439fd5e5b76e + diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/header-string-to-sign.txt new file mode 100644 index 00000000..06059dd1 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +80cb39203d058af815de2b79250ff56e1b73eb9b4718c86556cdc6f150c5d209 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/request.txt b/vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/request.txt new file mode 100644 index 00000000..25bc21e3 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slash-unnormalized/request.txt @@ -0,0 +1,2 @@ +GET // HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/context.json b/vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/header-canonical-request.txt new file mode 100644 index 00000000..2bdaf747 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/example/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/header-signature.txt new file mode 100644 index 00000000..71c7bac1 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/header-signature.txt @@ -0,0 +1 @@ +9a624bd73a37c9a373b5312afbebe7a714a789de108f0bdfe846570885f57e84 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/header-signed-request.txt new file mode 100644 index 00000000..0add1f15 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/header-signed-request.txt @@ -0,0 +1,5 @@ +GET //example// HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=9a624bd73a37c9a373b5312afbebe7a714a789de108f0bdfe846570885f57e84 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/header-string-to-sign.txt new file mode 100644 index 00000000..95d1fc25 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +cb96b4ac96d501f7c5c15bc6d67b3035061cfced4af6585ad927f7e6c985c015 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/request.txt b/vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/request.txt new file mode 100644 index 00000000..12e09319 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slashes-normalized/request.txt @@ -0,0 +1,2 @@ +GET //example// HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/context.json b/vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/context.json new file mode 100644 index 00000000..7bee5c78 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": false, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/header-canonical-request.txt new file mode 100644 index 00000000..dfffaafb --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +//example// + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/header-signature.txt new file mode 100644 index 00000000..edbcc0d7 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/header-signature.txt @@ -0,0 +1 @@ +87cca117541a147f6df867677d98a7d80dff226d2bfca9e4ffa899665623c7e5 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/header-signed-request.txt new file mode 100644 index 00000000..4cc12ddb --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/header-signed-request.txt @@ -0,0 +1,5 @@ +GET //example// HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=87cca117541a147f6df867677d98a7d80dff226d2bfca9e4ffa899665623c7e5 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/header-string-to-sign.txt new file mode 100644 index 00000000..3eba3905 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +528ec3105ee1f34ab014bb0a1a45da0ed2742a4fea3555149e5b4d5d201eb240 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/request.txt b/vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/request.txt new file mode 100644 index 00000000..12e09319 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-slashes-unnormalized/request.txt @@ -0,0 +1,2 @@ +GET //example// HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-space-normalized/context.json b/vk-registry/src/s3/testdata/sigv4/get-space-normalized/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-space-normalized/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-space-normalized/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-space-normalized/header-canonical-request.txt new file mode 100644 index 00000000..124a7096 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-space-normalized/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/example%20space/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-space-normalized/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-space-normalized/header-signature.txt new file mode 100644 index 00000000..f63892bb --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-space-normalized/header-signature.txt @@ -0,0 +1 @@ +652487583200325589f1fba4c7e578f72c47cb61beeca81406b39ddec1366741 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-space-normalized/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-space-normalized/header-signed-request.txt new file mode 100644 index 00000000..19cd5cb5 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-space-normalized/header-signed-request.txt @@ -0,0 +1,5 @@ +GET /example space/ HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=652487583200325589f1fba4c7e578f72c47cb61beeca81406b39ddec1366741 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-space-normalized/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-space-normalized/header-string-to-sign.txt new file mode 100644 index 00000000..a633f0c0 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-space-normalized/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +63ee75631ed7234ae61b5f736dfc7754cdccfedbff4b5128a915706ee9390d86 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-space-normalized/request.txt b/vk-registry/src/s3/testdata/sigv4/get-space-normalized/request.txt new file mode 100644 index 00000000..ab509ab7 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-space-normalized/request.txt @@ -0,0 +1,2 @@ +GET /example space/ HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/context.json b/vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/context.json new file mode 100644 index 00000000..7bee5c78 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": false, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/header-canonical-request.txt new file mode 100644 index 00000000..124a7096 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/example%20space/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/header-signature.txt new file mode 100644 index 00000000..f63892bb --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/header-signature.txt @@ -0,0 +1 @@ +652487583200325589f1fba4c7e578f72c47cb61beeca81406b39ddec1366741 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/header-signed-request.txt new file mode 100644 index 00000000..19cd5cb5 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/header-signed-request.txt @@ -0,0 +1,5 @@ +GET /example space/ HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=652487583200325589f1fba4c7e578f72c47cb61beeca81406b39ddec1366741 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/header-string-to-sign.txt new file mode 100644 index 00000000..a633f0c0 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +63ee75631ed7234ae61b5f736dfc7754cdccfedbff4b5128a915706ee9390d86 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/request.txt b/vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/request.txt new file mode 100644 index 00000000..ab509ab7 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-space-unnormalized/request.txt @@ -0,0 +1,2 @@ +GET /example space/ HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-unreserved/context.json b/vk-registry/src/s3/testdata/sigv4/get-unreserved/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-unreserved/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-unreserved/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-unreserved/header-canonical-request.txt new file mode 100644 index 00000000..8af54df2 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-unreserved/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-unreserved/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-unreserved/header-signature.txt new file mode 100644 index 00000000..e5033549 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-unreserved/header-signature.txt @@ -0,0 +1 @@ +07ef7494c76fa4850883e2b006601f940f8a34d404d0cfa977f52a65bbf5f24f \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-unreserved/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-unreserved/header-signed-request.txt new file mode 100644 index 00000000..43678920 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-unreserved/header-signed-request.txt @@ -0,0 +1,5 @@ +GET /-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=07ef7494c76fa4850883e2b006601f940f8a34d404d0cfa977f52a65bbf5f24f + diff --git a/vk-registry/src/s3/testdata/sigv4/get-unreserved/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-unreserved/header-string-to-sign.txt new file mode 100644 index 00000000..e9dc5414 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-unreserved/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +6a968768eefaa713e2a6b16b589a8ea192661f098f37349f4e2c0082757446f9 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-unreserved/request.txt b/vk-registry/src/s3/testdata/sigv4/get-unreserved/request.txt new file mode 100644 index 00000000..a149694b --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-unreserved/request.txt @@ -0,0 +1,2 @@ +GET /-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-utf8/context.json b/vk-registry/src/s3/testdata/sigv4/get-utf8/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-utf8/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-utf8/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-utf8/header-canonical-request.txt new file mode 100644 index 00000000..5d4b9f61 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-utf8/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/%E1%88%B4 + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-utf8/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-utf8/header-signature.txt new file mode 100644 index 00000000..9a136766 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-utf8/header-signature.txt @@ -0,0 +1 @@ +8318018e0b0f223aa2bbf98705b62bb787dc9c0e678f255a891fd03141be5d85 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-utf8/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-utf8/header-signed-request.txt new file mode 100644 index 00000000..5aaf47c3 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-utf8/header-signed-request.txt @@ -0,0 +1,5 @@ +GET /ሴ HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=8318018e0b0f223aa2bbf98705b62bb787dc9c0e678f255a891fd03141be5d85 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-utf8/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-utf8/header-string-to-sign.txt new file mode 100644 index 00000000..5edc8f45 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-utf8/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +2a0a97d02205e45ce2e994789806b19270cfbbb0921b278ccf58f5249ac42102 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-utf8/request.txt b/vk-registry/src/s3/testdata/sigv4/get-utf8/request.txt new file mode 100644 index 00000000..9e273f69 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-utf8/request.txt @@ -0,0 +1,2 @@ +GET /ሴ HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/context.json b/vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/header-canonical-request.txt new file mode 100644 index 00000000..c6cdceda --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/ +Param1=value1 +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/header-signature.txt new file mode 100644 index 00000000..d7f1c1c9 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/header-signature.txt @@ -0,0 +1 @@ +a67d582fa61cc504c4bae71f336f98b97f1ea3c7a6bfe1b6e45aec72011b9aeb \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/header-signed-request.txt new file mode 100644 index 00000000..1c607902 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/header-signed-request.txt @@ -0,0 +1,5 @@ +GET /?Param1=value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=a67d582fa61cc504c4bae71f336f98b97f1ea3c7a6bfe1b6e45aec72011b9aeb + diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/header-string-to-sign.txt new file mode 100644 index 00000000..c4ed216c --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +1e24db194ed7d0eec2de28d7369675a243488e08526e8c1c73571282f7c517ab \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/request.txt new file mode 100644 index 00000000..f4a03bb0 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-empty-query-key/request.txt @@ -0,0 +1,2 @@ +GET /?Param1=value1 HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/context.json b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/header-canonical-request.txt new file mode 100644 index 00000000..0c8ba21f --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/ +%E1%88%B4=Value1&Param=Value2&Param-3=Value3 +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/header-signature.txt new file mode 100644 index 00000000..9ab65edb --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/header-signature.txt @@ -0,0 +1 @@ +371d3713e185cc334048618a97f809c9ffe339c62934c032af5a0e595648fcac \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/header-signed-request.txt new file mode 100644 index 00000000..0bcdf987 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/header-signed-request.txt @@ -0,0 +1,5 @@ +GET /?Param-3=Value3&Param=Value2&%E1%88%B4=Value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=371d3713e185cc334048618a97f809c9ffe339c62934c032af5a0e595648fcac + diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/header-string-to-sign.txt new file mode 100644 index 00000000..bf674ad6 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +868294f5c38bd141c4972a373a76654f1418a8e4fc18b2e7903ae45e8ae0ec71 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/request.txt new file mode 100644 index 00000000..368c6e3d --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-encoded/request.txt @@ -0,0 +1,2 @@ +GET /?Param-3=Value3&Param=Value2&%E1%88%B4=Value1 HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/context.json b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/header-canonical-request.txt new file mode 100644 index 00000000..8ae02cd6 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/ +Param1=value1&Param2=value2 +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/header-signature.txt new file mode 100644 index 00000000..6a8cb9d1 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/header-signature.txt @@ -0,0 +1 @@ +b97d918cfa904a5beff61c982a1b6f458b799221646efd99d3219ec94cdf2500 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/header-signed-request.txt new file mode 100644 index 00000000..f78a9d2f --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/header-signed-request.txt @@ -0,0 +1,5 @@ +GET /?Param2=value2&Param1=value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=b97d918cfa904a5beff61c982a1b6f458b799221646efd99d3219ec94cdf2500 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/header-string-to-sign.txt new file mode 100644 index 00000000..f773de59 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +816cd5b414d056048ba4f7c5386d6e0533120fb1fcfa93762cf0fc39e2cf19e0 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/request.txt new file mode 100644 index 00000000..1158ac4e --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-order-key-case/request.txt @@ -0,0 +1,2 @@ +GET /?Param2=value2&Param1=value1 HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/context.json b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/header-canonical-request.txt new file mode 100644 index 00000000..5249be3b --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/ +-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz=-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/header-signature.txt new file mode 100644 index 00000000..876a64b9 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/header-signature.txt @@ -0,0 +1 @@ +9c3e54bfcdf0b19771a7f523ee5669cdf59bc7cc0884027167c21bb143a40197 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/header-signed-request.txt new file mode 100644 index 00000000..9d0d9932 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/header-signed-request.txt @@ -0,0 +1,5 @@ +GET /?-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz=-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=9c3e54bfcdf0b19771a7f523ee5669cdf59bc7cc0884027167c21bb143a40197 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/header-string-to-sign.txt new file mode 100644 index 00000000..24a97d20 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +c30d4703d9f799439be92736156d47ccfb2d879ddf56f5befa6d1d6aab979177 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/request.txt new file mode 100644 index 00000000..36b2fe3e --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query-unreserved/request.txt @@ -0,0 +1,2 @@ +GET /?-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz=-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query/context.json b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query/header-canonical-request.txt new file mode 100644 index 00000000..ed91561f --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query/header-signature.txt new file mode 100644 index 00000000..87f295fb --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query/header-signature.txt @@ -0,0 +1 @@ +5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query/header-signed-request.txt new file mode 100644 index 00000000..36b86e79 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query/header-signed-request.txt @@ -0,0 +1,5 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query/header-string-to-sign.txt new file mode 100644 index 00000000..b187649c --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-query/request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query/request.txt new file mode 100644 index 00000000..e659c3be --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-query/request.txt @@ -0,0 +1,2 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/context.json b/vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/header-canonical-request.txt new file mode 100644 index 00000000..a835c9e4 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/ +%E1%88%B4=bar +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/header-signature.txt new file mode 100644 index 00000000..19a2cc99 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/header-signature.txt @@ -0,0 +1 @@ +2cdec8eed098649ff3a119c94853b13c643bcf08f8b0a1d91e12c9027818dd04 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/header-signed-request.txt new file mode 100644 index 00000000..acb13eeb --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/header-signed-request.txt @@ -0,0 +1,5 @@ +GET /?ሴ=bar HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=2cdec8eed098649ff3a119c94853b13c643bcf08f8b0a1d91e12c9027818dd04 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/header-string-to-sign.txt new file mode 100644 index 00000000..51ee71b7 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +eb30c5bed55734080471a834cc727ae56beb50e5f39d1bff6d0d38cb192a7073 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/request.txt new file mode 100644 index 00000000..aff333aa --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-utf8-query/request.txt @@ -0,0 +1,2 @@ +GET /?ሴ=bar HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/context.json b/vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/context.json new file mode 100644 index 00000000..a334e18c --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/context.json @@ -0,0 +1,13 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + "token": "6e86291e8372ff2a2260956d9b8aae1d763fbf315fa00fa31553b73ebf194267" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/header-canonical-request.txt new file mode 100644 index 00000000..ccacdeb4 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/header-canonical-request.txt @@ -0,0 +1,9 @@ +GET +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z +x-amz-security-token:6e86291e8372ff2a2260956d9b8aae1d763fbf315fa00fa31553b73ebf194267 + +host;x-amz-date;x-amz-security-token +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/header-signature.txt new file mode 100644 index 00000000..083e30d5 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/header-signature.txt @@ -0,0 +1 @@ +07ec1639c89043aa0e3e2de82b96708f198cceab042d4a97044c66dd9f74e7f8 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/header-signed-request.txt new file mode 100644 index 00000000..52879328 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/header-signed-request.txt @@ -0,0 +1,6 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Security-Token:6e86291e8372ff2a2260956d9b8aae1d763fbf315fa00fa31553b73ebf194267 +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=07ec1639c89043aa0e3e2de82b96708f198cceab042d4a97044c66dd9f74e7f8 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/header-string-to-sign.txt new file mode 100644 index 00000000..742b880c --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +067b36aa60031588cea4a4cde1f21215227a047690c72247f1d70b32fbbfad2b \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/request.txt new file mode 100644 index 00000000..e659c3be --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla-with-session-token/request.txt @@ -0,0 +1,2 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla/context.json b/vk-registry/src/s3/testdata/sigv4/get-vanilla/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla/header-canonical-request.txt new file mode 100644 index 00000000..ed91561f --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla/header-canonical-request.txt @@ -0,0 +1,8 @@ +GET +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla/header-signature.txt new file mode 100644 index 00000000..87f295fb --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla/header-signature.txt @@ -0,0 +1 @@ +5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla/header-signed-request.txt new file mode 100644 index 00000000..36b86e79 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla/header-signed-request.txt @@ -0,0 +1,5 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 + diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla/header-string-to-sign.txt new file mode 100644 index 00000000..b187649c --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/get-vanilla/request.txt b/vk-registry/src/s3/testdata/sigv4/get-vanilla/request.txt new file mode 100644 index 00000000..e659c3be --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/get-vanilla/request.txt @@ -0,0 +1,2 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/post-header-key-case/context.json b/vk-registry/src/s3/testdata/sigv4/post-header-key-case/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-header-key-case/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-header-key-case/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/post-header-key-case/header-canonical-request.txt new file mode 100644 index 00000000..5c3a9434 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-header-key-case/header-canonical-request.txt @@ -0,0 +1,8 @@ +POST +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-header-key-case/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/post-header-key-case/header-signature.txt new file mode 100644 index 00000000..629487fd --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-header-key-case/header-signature.txt @@ -0,0 +1 @@ +5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-header-key-case/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/post-header-key-case/header-signed-request.txt new file mode 100644 index 00000000..0d99b1c8 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-header-key-case/header-signed-request.txt @@ -0,0 +1,5 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b + diff --git a/vk-registry/src/s3/testdata/sigv4/post-header-key-case/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/post-header-key-case/header-string-to-sign.txt new file mode 100644 index 00000000..a6367039 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-header-key-case/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +553f88c9e4d10fc9e109e2aeb65f030801b70c2f6468faca261d401ae622fc87 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-header-key-case/request.txt b/vk-registry/src/s3/testdata/sigv4/post-header-key-case/request.txt new file mode 100644 index 00000000..3f0a82a2 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-header-key-case/request.txt @@ -0,0 +1,2 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/post-header-key-sort/context.json b/vk-registry/src/s3/testdata/sigv4/post-header-key-sort/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-header-key-sort/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-header-key-sort/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/post-header-key-sort/header-canonical-request.txt new file mode 100644 index 00000000..ebe943e8 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-header-key-sort/header-canonical-request.txt @@ -0,0 +1,9 @@ +POST +/ + +host:example.amazonaws.com +my-header1:value1 +x-amz-date:20150830T123600Z + +host;my-header1;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-header-key-sort/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/post-header-key-sort/header-signature.txt new file mode 100644 index 00000000..e4e64285 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-header-key-sort/header-signature.txt @@ -0,0 +1 @@ +c5410059b04c1ee005303aed430f6e6645f61f4dc9e1461ec8f8916fdf18852c \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-header-key-sort/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/post-header-key-sort/header-signed-request.txt new file mode 100644 index 00000000..e3778f5f --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-header-key-sort/header-signed-request.txt @@ -0,0 +1,6 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:value1 +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=c5410059b04c1ee005303aed430f6e6645f61f4dc9e1461ec8f8916fdf18852c + diff --git a/vk-registry/src/s3/testdata/sigv4/post-header-key-sort/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/post-header-key-sort/header-string-to-sign.txt new file mode 100644 index 00000000..eb663626 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-header-key-sort/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +9368318c2967cf6de74404b30c65a91e8f6253e0a8659d6d5319f1a812f87d65 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-header-key-sort/request.txt b/vk-registry/src/s3/testdata/sigv4/post-header-key-sort/request.txt new file mode 100644 index 00000000..917720ef --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-header-key-sort/request.txt @@ -0,0 +1,3 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:value1 diff --git a/vk-registry/src/s3/testdata/sigv4/post-header-value-case/context.json b/vk-registry/src/s3/testdata/sigv4/post-header-value-case/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-header-value-case/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-header-value-case/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/post-header-value-case/header-canonical-request.txt new file mode 100644 index 00000000..af824c88 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-header-value-case/header-canonical-request.txt @@ -0,0 +1,9 @@ +POST +/ + +host:example.amazonaws.com +my-header1:VALUE1 +x-amz-date:20150830T123600Z + +host;my-header1;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-header-value-case/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/post-header-value-case/header-signature.txt new file mode 100644 index 00000000..bb3809cf --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-header-value-case/header-signature.txt @@ -0,0 +1 @@ +cdbc9802e29d2942e5e10b5bccfdd67c5f22c7c4e8ae67b53629efa58b974b7d \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-header-value-case/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/post-header-value-case/header-signed-request.txt new file mode 100644 index 00000000..10f67615 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-header-value-case/header-signed-request.txt @@ -0,0 +1,6 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:VALUE1 +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=cdbc9802e29d2942e5e10b5bccfdd67c5f22c7c4e8ae67b53629efa58b974b7d + diff --git a/vk-registry/src/s3/testdata/sigv4/post-header-value-case/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/post-header-value-case/header-string-to-sign.txt new file mode 100644 index 00000000..40062c79 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-header-value-case/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +d51ced243e649e3de6ef63afbbdcbca03131a21a7103a1583706a64618606a93 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-header-value-case/request.txt b/vk-registry/src/s3/testdata/sigv4/post-header-value-case/request.txt new file mode 100644 index 00000000..5f14c915 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-header-value-case/request.txt @@ -0,0 +1,3 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:VALUE1 diff --git a/vk-registry/src/s3/testdata/sigv4/post-sts-header-after/context.json b/vk-registry/src/s3/testdata/sigv4/post-sts-header-after/context.json new file mode 100644 index 00000000..8f1d5ced --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-sts-header-after/context.json @@ -0,0 +1,14 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + "token": "AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z", + "omit_session_token": true +} diff --git a/vk-registry/src/s3/testdata/sigv4/post-sts-header-after/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/post-sts-header-after/header-canonical-request.txt new file mode 100644 index 00000000..5c3a9434 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-sts-header-after/header-canonical-request.txt @@ -0,0 +1,8 @@ +POST +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-sts-header-after/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/post-sts-header-after/header-signature.txt new file mode 100644 index 00000000..629487fd --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-sts-header-after/header-signature.txt @@ -0,0 +1 @@ +5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-sts-header-after/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/post-sts-header-after/header-signed-request.txt new file mode 100644 index 00000000..f96ff9e1 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-sts-header-after/header-signed-request.txt @@ -0,0 +1,6 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Security-Token:AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA== +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b + diff --git a/vk-registry/src/s3/testdata/sigv4/post-sts-header-after/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/post-sts-header-after/header-string-to-sign.txt new file mode 100644 index 00000000..a6367039 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-sts-header-after/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +553f88c9e4d10fc9e109e2aeb65f030801b70c2f6468faca261d401ae622fc87 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-sts-header-after/request.txt b/vk-registry/src/s3/testdata/sigv4/post-sts-header-after/request.txt new file mode 100644 index 00000000..3f0a82a2 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-sts-header-after/request.txt @@ -0,0 +1,2 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/post-sts-header-before/context.json b/vk-registry/src/s3/testdata/sigv4/post-sts-header-before/context.json new file mode 100644 index 00000000..5187392c --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-sts-header-before/context.json @@ -0,0 +1,14 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + "token": "AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z", + "omit_session_token": false +} diff --git a/vk-registry/src/s3/testdata/sigv4/post-sts-header-before/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/post-sts-header-before/header-canonical-request.txt new file mode 100644 index 00000000..1d5a462e --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-sts-header-before/header-canonical-request.txt @@ -0,0 +1,9 @@ +POST +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z +x-amz-security-token:AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA== + +host;x-amz-date;x-amz-security-token +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-sts-header-before/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/post-sts-header-before/header-signature.txt new file mode 100644 index 00000000..85a7c29c --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-sts-header-before/header-signature.txt @@ -0,0 +1 @@ +85d96828115b5dc0cfc3bd16ad9e210dd772bbebba041836c64533a82be05ead \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-sts-header-before/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/post-sts-header-before/header-signed-request.txt new file mode 100644 index 00000000..4d0b1435 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-sts-header-before/header-signed-request.txt @@ -0,0 +1,6 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Security-Token:AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA== +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=85d96828115b5dc0cfc3bd16ad9e210dd772bbebba041836c64533a82be05ead + diff --git a/vk-registry/src/s3/testdata/sigv4/post-sts-header-before/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/post-sts-header-before/header-string-to-sign.txt new file mode 100644 index 00000000..bc39ccfc --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-sts-header-before/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +c237e1b440d4c63c32ca95b5b99481081cb7b13c7e40434868e71567c1a882f6 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-sts-header-before/request.txt b/vk-registry/src/s3/testdata/sigv4/post-sts-header-before/request.txt new file mode 100644 index 00000000..3f0a82a2 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-sts-header-before/request.txt @@ -0,0 +1,2 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/context.json b/vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/header-canonical-request.txt new file mode 100644 index 00000000..f5058d43 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/header-canonical-request.txt @@ -0,0 +1,8 @@ +POST +/ +Param1=value1 +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/header-signature.txt new file mode 100644 index 00000000..80ed4337 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/header-signature.txt @@ -0,0 +1 @@ +28038455d6de14eafc1f9222cf5aa6f1a96197d7deb8263271d420d138af7f11 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/header-signed-request.txt new file mode 100644 index 00000000..4e614f88 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/header-signed-request.txt @@ -0,0 +1,5 @@ +POST /?Param1=value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=28038455d6de14eafc1f9222cf5aa6f1a96197d7deb8263271d420d138af7f11 + diff --git a/vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/header-string-to-sign.txt new file mode 100644 index 00000000..ca7cc661 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +9d659678c1756bb3113e2ce898845a0a79dbbc57b740555917687f1b3340fbbd \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/request.txt b/vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/request.txt new file mode 100644 index 00000000..c9d6e5df --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-vanilla-empty-query-value/request.txt @@ -0,0 +1,2 @@ +POST /?Param1=value1 HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/post-vanilla-query/context.json b/vk-registry/src/s3/testdata/sigv4/post-vanilla-query/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-vanilla-query/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-vanilla-query/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/post-vanilla-query/header-canonical-request.txt new file mode 100644 index 00000000..f5058d43 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-vanilla-query/header-canonical-request.txt @@ -0,0 +1,8 @@ +POST +/ +Param1=value1 +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-vanilla-query/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/post-vanilla-query/header-signature.txt new file mode 100644 index 00000000..80ed4337 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-vanilla-query/header-signature.txt @@ -0,0 +1 @@ +28038455d6de14eafc1f9222cf5aa6f1a96197d7deb8263271d420d138af7f11 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-vanilla-query/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/post-vanilla-query/header-signed-request.txt new file mode 100644 index 00000000..4e614f88 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-vanilla-query/header-signed-request.txt @@ -0,0 +1,5 @@ +POST /?Param1=value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=28038455d6de14eafc1f9222cf5aa6f1a96197d7deb8263271d420d138af7f11 + diff --git a/vk-registry/src/s3/testdata/sigv4/post-vanilla-query/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/post-vanilla-query/header-string-to-sign.txt new file mode 100644 index 00000000..ca7cc661 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-vanilla-query/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +9d659678c1756bb3113e2ce898845a0a79dbbc57b740555917687f1b3340fbbd \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-vanilla-query/request.txt b/vk-registry/src/s3/testdata/sigv4/post-vanilla-query/request.txt new file mode 100644 index 00000000..c9d6e5df --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-vanilla-query/request.txt @@ -0,0 +1,2 @@ +POST /?Param1=value1 HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/post-vanilla/context.json b/vk-registry/src/s3/testdata/sigv4/post-vanilla/context.json new file mode 100644 index 00000000..0468d800 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-vanilla/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": false, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-vanilla/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/post-vanilla/header-canonical-request.txt new file mode 100644 index 00000000..5c3a9434 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-vanilla/header-canonical-request.txt @@ -0,0 +1,8 @@ +POST +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-vanilla/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/post-vanilla/header-signature.txt new file mode 100644 index 00000000..629487fd --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-vanilla/header-signature.txt @@ -0,0 +1 @@ +5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-vanilla/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/post-vanilla/header-signed-request.txt new file mode 100644 index 00000000..0d99b1c8 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-vanilla/header-signed-request.txt @@ -0,0 +1,5 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b + diff --git a/vk-registry/src/s3/testdata/sigv4/post-vanilla/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/post-vanilla/header-string-to-sign.txt new file mode 100644 index 00000000..a6367039 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-vanilla/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +553f88c9e4d10fc9e109e2aeb65f030801b70c2f6468faca261d401ae622fc87 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-vanilla/request.txt b/vk-registry/src/s3/testdata/sigv4/post-vanilla/request.txt new file mode 100644 index 00000000..3f0a82a2 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-vanilla/request.txt @@ -0,0 +1,2 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com diff --git a/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/context.json b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/context.json new file mode 100644 index 00000000..0db1df10 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": true, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/header-canonical-request.txt new file mode 100644 index 00000000..c218fdf2 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/header-canonical-request.txt @@ -0,0 +1,11 @@ +POST +/ + +content-length:13 +content-type:application/x-www-form-urlencoded; charset=utf-8 +host:example.amazonaws.com +x-amz-content-sha256:9095672bbd1f56dfc5b65f3e153adc8731a4a654192329106275f4c7b24d0b6e +x-amz-date:20150830T123600Z + +content-length;content-type;host;x-amz-content-sha256;x-amz-date +9095672bbd1f56dfc5b65f3e153adc8731a4a654192329106275f4c7b24d0b6e \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/header-signature.txt new file mode 100644 index 00000000..836aa8b9 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/header-signature.txt @@ -0,0 +1 @@ +328d1b9eaadca9f5818ef05e8392801e091653bafec24fcab71e7344e7f51422 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/header-signed-request.txt new file mode 100644 index 00000000..9262d53b --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/header-signed-request.txt @@ -0,0 +1,9 @@ +POST / HTTP/1.1 +Content-Type:application/x-www-form-urlencoded; charset=utf-8 +Host:example.amazonaws.com +Content-Length:13 +X-Amz-Date:20150830T123600Z +x-amz-content-sha256:9095672bbd1f56dfc5b65f3e153adc8731a4a654192329106275f4c7b24d0b6e +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-content-sha256;x-amz-date, Signature=328d1b9eaadca9f5818ef05e8392801e091653bafec24fcab71e7344e7f51422 + +Param1=value1 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/header-string-to-sign.txt new file mode 100644 index 00000000..ddbdccc4 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +a89f1a5b53e37702ee6363ce1da3ce8f54386f3c8f352ae652153c2982a0bc4d \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/request.txt b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/request.txt new file mode 100644 index 00000000..8289eaa5 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded-parameters/request.txt @@ -0,0 +1,6 @@ +POST / HTTP/1.1 +Content-Type:application/x-www-form-urlencoded; charset=utf-8 +Host:example.amazonaws.com +Content-Length:13 + +Param1=value1 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/context.json b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/context.json new file mode 100644 index 00000000..0db1df10 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/context.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_key_id": "AKIDEXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }, + "expiration_in_seconds": 3600, + "normalize": true, + "region": "us-east-1", + "service": "service", + "sign_body": true, + "timestamp": "2015-08-30T12:36:00Z" +} \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/header-canonical-request.txt b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/header-canonical-request.txt new file mode 100644 index 00000000..4d116a56 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/header-canonical-request.txt @@ -0,0 +1,11 @@ +POST +/ + +content-length:13 +content-type:application/x-www-form-urlencoded +host:example.amazonaws.com +x-amz-content-sha256:9095672bbd1f56dfc5b65f3e153adc8731a4a654192329106275f4c7b24d0b6e +x-amz-date:20150830T123600Z + +content-length;content-type;host;x-amz-content-sha256;x-amz-date +9095672bbd1f56dfc5b65f3e153adc8731a4a654192329106275f4c7b24d0b6e \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/header-signature.txt b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/header-signature.txt new file mode 100644 index 00000000..0d2ac374 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/header-signature.txt @@ -0,0 +1 @@ +d3875051da38690788ef43de4db0d8f280229d82040bfac253562e56c3f20e0b \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/header-signed-request.txt b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/header-signed-request.txt new file mode 100644 index 00000000..bd5e49c7 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/header-signed-request.txt @@ -0,0 +1,9 @@ +POST / HTTP/1.1 +Content-Type:application/x-www-form-urlencoded +Host:example.amazonaws.com +Content-Length:13 +X-Amz-Date:20150830T123600Z +x-amz-content-sha256:9095672bbd1f56dfc5b65f3e153adc8731a4a654192329106275f4c7b24d0b6e +Authorization:AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-content-sha256;x-amz-date, Signature=d3875051da38690788ef43de4db0d8f280229d82040bfac253562e56c3f20e0b + +Param1=value1 \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/header-string-to-sign.txt b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/header-string-to-sign.txt new file mode 100644 index 00000000..f1c599c1 --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/header-string-to-sign.txt @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +b1edd1d03544c25390e32085d55b57acc9a3961bb59415ff86c45c3d89d16cfb \ No newline at end of file diff --git a/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/request.txt b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/request.txt new file mode 100644 index 00000000..760b2d3c --- /dev/null +++ b/vk-registry/src/s3/testdata/sigv4/post-x-www-form-urlencoded/request.txt @@ -0,0 +1,6 @@ +POST / HTTP/1.1 +Content-Type:application/x-www-form-urlencoded +Host:example.amazonaws.com +Content-Length:13 + +Param1=value1 \ No newline at end of file diff --git a/vk-registry/src/s3/xml.rs b/vk-registry/src/s3/xml.rs new file mode 100644 index 00000000..6217ecf7 --- /dev/null +++ b/vk-registry/src/s3/xml.rs @@ -0,0 +1,74 @@ +//! The XML S3 answers in: the `Error` document every refusal carries. + +use bytes::Bytes; +use hyper::header::CONTENT_TYPE; +use hyper::{Response, StatusCode}; + +use crate::{Body, body_of}; + +/// Escape a string for XML text: [`crate::html_escape`]'s five characters are XML's too. +/// What reaches it is a request path, a header value or a message of ours, none of which +/// can hold a control character XML 1.0 has no way to write. +pub(crate) fn escape(s: &str) -> String { + crate::html_escape(s) +} + +/// An S3 `Error` document: `code`, `message`, then each of `extra` as an element of its +/// own (`ServerTime` and the like), the `resource` the request named and its `request_id`. +/// Every string is escaped. +pub(crate) fn error( + status: StatusCode, + code: &str, + message: &str, + extra: &[(&str, &str)], + resource: &str, + request_id: &str, +) -> Response { + let mut body = format!( + "{}{}", + escape(code), + escape(message) + ); + for (name, value) in extra { + body.push_str(&format!("<{name}>{}", escape(value))); + } + body.push_str(&format!( + "{}{}", + escape(resource), + escape(request_id) + )); + Response::builder() + .status(status) + .header(CONTENT_TYPE, "application/xml") + .body(body_of(Bytes::from(body))) + .expect("building an S3 error") +} + +#[cfg(test)] +mod tests { + use super::*; + use http_body_util::BodyExt as _; + + #[tokio::test] + async fn every_caller_string_is_escaped() { + let resp = error( + StatusCode::FORBIDDEN, + "AccessDenied", + "a & 'c' \"d\"", + &[("RequestTime", "")], + "/s3/", + "0123456789abcdef", + ); + assert_eq!(resp.status(), 403); + assert_eq!(resp.headers()[CONTENT_TYPE], "application/xml"); + let body = resp.into_body().collect().await.unwrap().to_bytes(); + assert_eq!( + std::str::from_utf8(&body).unwrap(), + "AccessDenied\ + a <b> & 'c' "d"\ + <t>\ + /s3/</Resource><x>\ + 0123456789abcdef" + ); + } +} diff --git a/vk-registry/tests/s3_e2e.rs b/vk-registry/tests/s3_e2e.rs new file mode 100644 index 00000000..b2ea66ef --- /dev/null +++ b/vk-registry/tests/s3_e2e.rs @@ -0,0 +1,809 @@ +//! S3 over real HTTP: the gate `/s3/` requests pass, under every authentication mode the +//! server has. SigV4 requests are signed here by a signer of the tests' own, checked against +//! AWS's published example; responses are read off a raw socket, so a test sees exactly what +//! a connection carries and can send a second request on it. + +use std::sync::Arc; +use std::time::{Duration, SystemTime}; + +use hmac::{KeyInit as _, Mac as _}; +use sha2::Digest as _; +use tokio::io::{AsyncBufReadExt as _, AsyncReadExt as _, AsyncWriteExt as _, BufReader}; +use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; +use vk_hub_proto::token::{Claims, Grant}; +use vk_peer::token::Signer; +use vk_registry::accounts::{Action, Db, EmailUpdate, Scope}; +use vk_registry::auth::Auth; +use vk_registry::config::{AuthMode, OidcSpec}; +use vk_registry::lock::LockManager; +use vk_registry::{Authenticator, ServerConfig, ServerState, Store}; + +const AUD: &str = "9c2f4e1a7b3d5e60f1a2b3c4d5e6f708"; +const UNSIGNED: &str = "UNSIGNED-PAYLOAD"; + +fn tmp(tag: &str) -> std::path::PathBuf { + let p = std::env::temp_dir().join(format!( + "vk-registry-s3-e2e-{tag}-{}-{:?}", + std::process::id(), + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let _ = std::fs::remove_dir_all(&p); + std::fs::create_dir_all(&p).unwrap(); + p +} + +/// Run serve_on in a separate thread with an ephemeral listener; the address it took. +fn spawn(state: Arc) -> String { + let _ = rustls::crypto::ring::default_provider().install_default(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let addr = listener.local_addr().unwrap(); + std::thread::spawn(move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(async move { + let l = tokio::net::TcpListener::from_std(listener).unwrap(); + let _ = vk_registry::serve_on(l, state).await; + }); + }); + addr.to_string() +} + +/// A server with no credential configured; `s3` as given. +fn open_state(dir: &std::path::Path, s3: bool) -> Arc { + let mut cfg = ServerConfig::local("127.0.0.1:5000".parse().unwrap(), dir.join("store")); + cfg.s3 = s3; + Arc::new(cfg.into_state().expect("a local config starts")) +} + +/// A shared-secret server with `auth`, taking hub tokens signed by a fresh key. +fn shared(dir: &std::path::Path, auth: Auth, s3: bool) -> (Arc, Signer) { + let signer = Signer::generate(&dir.join("token-key")).unwrap(); + let state = Arc::new(ServerState { + store: Arc::new(Store::new(dir.join("store")).unwrap()), + upstreams: vec![], + locks: LockManager::new(), + auth: Authenticator::Shared(auth), + tls: None, + webdav: true, + s3, + tokens: Default::default(), + handover: Default::default(), + }); + state.tokens.open(AUD, vec![signer.public().clone()]); + (state, signer) +} + +/// Accounts-mode state built the way `serve` builds it, as in `dav_e2e.rs`. +fn accounts_state(dir: &std::path::Path) -> Arc { + let secret = dir.join("oidc-secret"); + std::fs::write(&secret, "s3cr3t\n").unwrap(); + let mut cfg = ServerConfig::local("127.0.0.1:5000".parse().unwrap(), dir.join("store")); + cfg.mode = AuthMode::Accounts; + cfg.oidc = Some(OidcSpec { + issuer: "https://login.example.com".to_string(), + client_id: "vk-registry".to_string(), + client_secret_file: secret, + public_url: "https://registry.internal".to_string(), + }); + Arc::new(cfg.into_state().expect("a valid accounts config starts")) +} + +fn accounts_db(state: &ServerState) -> &Db { + match &state.auth { + Authenticator::Accounts { db, .. } => db, + _ => panic!("not accounts mode"), + } +} + +fn hub_claims(aud: &str, iat: u64, exp: u64) -> Claims { + Claims { + kid: String::new(), + aud: aud.into(), + sub: "job:j1".into(), + jti: vk_peer::token::new_jti().unwrap(), + pipeline: None, + iat, + exp, + grants: vec![Grant { + prefix: "p/gl/42".into(), + read: true, + write: true, + lock: false, + }], + } +} + +// ---- the test-side SigV4 signer ---- + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +fn hmac(key: &[u8], data: &str) -> Vec { + let mut mac = hmac::Hmac::::new_from_slice(key).unwrap(); + mac.update(data.as_bytes()); + mac.finalize().into_bytes().to_vec() +} + +/// `secs` since the epoch as SigV4's `YYYYMMDDTHHMMSSZ`. +fn stamp(secs: u64) -> String { + let (days, rem) = ((secs / 86_400) as i64, secs % 86_400); + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = yoe + era * 400 + i64::from(m <= 2); + let (h, mi, s) = (rem / 3600, rem / 60 % 60, rem % 60); + format!("{y:04}{m:02}{d:02}T{h:02}{mi:02}{s:02}Z") +} + +/// The `Authorization` value for a request with an already canonical `query` and `headers` +/// (lowercase names, sorted, every one signed), signed by `id` with `secret` at `at`. +fn authorization( + method: &str, + path: &str, + query: &str, + headers: &[(&str, &str)], + payload: &str, + (id, secret): (&str, &str), + at: &str, +) -> String { + let date = &at[..8]; + let scope = format!("{date}/us-east-1/s3/aws4_request"); + let names: Vec<&str> = headers.iter().map(|(n, _)| *n).collect(); + let mut canonical = format!("{method}\n{path}\n{query}\n"); + for (n, v) in headers { + canonical.push_str(&format!("{n}:{}\n", v.trim())); + } + canonical.push_str(&format!("\n{}\n{payload}", names.join(";"))); + let sts = format!( + "AWS4-HMAC-SHA256\n{at}\n{scope}\n{}", + hex(&sha2::Sha256::digest(canonical.as_bytes())) + ); + let mut key = hmac(format!("AWS4{secret}").as_bytes(), date); + for part in ["us-east-1", "s3", "aws4_request"] { + key = hmac(&key, part); + } + format!( + "AWS4-HMAC-SHA256 Credential={id}/{scope}, SignedHeaders={}, Signature={}", + names.join(";"), + hex(&hmac(&key, &sts)) + ) +} + +/// The signer reproduces AWS's GET Object example, signature and all. +#[test] +fn the_test_signer_reproduces_the_aws_get_object_example() { + let empty = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + let at = "20130524T000000Z"; + let auth = authorization( + "GET", + "/test.txt", + "", + &[ + ("host", "examplebucket.s3.amazonaws.com"), + ("range", "bytes=0-9"), + ("x-amz-content-sha256", empty), + ("x-amz-date", at), + ], + empty, + ( + "AKIAIOSFODNN7EXAMPLE", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + ), + at, + ); + assert_eq!( + auth, + "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, \ + SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, \ + Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41" + ); + assert_eq!(stamp(1_369_353_600), at); +} + +// ---- requests over a raw connection ---- + +/// One request: its line, its headers beyond `Host`, its body. +struct Req { + method: &'static str, + path: String, + query: String, + headers: Vec<(String, String)>, + body: Vec, +} + +impl Req { + fn new(method: &'static str, path: &str) -> Self { + Req { + method, + path: path.to_string(), + query: String::new(), + headers: Vec::new(), + body: Vec::new(), + } + } + + fn query(mut self, q: &str) -> Self { + self.query = q.to_string(); + self + } + + fn header(mut self, name: &str, value: &str) -> Self { + self.headers.push((name.to_string(), value.to_string())); + self + } + + fn body(mut self, body: Vec) -> Self { + self.body = body; + self + } + + /// Signed for `host` by `key` (id, secret), `skew` seconds off now (UNSIGNED-PAYLOAD). + fn signed(self, host: &str, key: (&str, &str), skew: i64) -> Self { + let at = stamp(vk_peer::now_secs().saturating_add_signed(skew)); + let auth = authorization( + self.method, + &self.path, + &self.query, + &[ + ("host", host), + ("x-amz-content-sha256", UNSIGNED), + ("x-amz-date", &at), + ], + UNSIGNED, + key, + &at, + ); + self.header("x-amz-content-sha256", UNSIGNED) + .header("x-amz-date", &at) + .header("Authorization", &auth) + } + + fn head(&self, host: &str) -> String { + let target = if self.query.is_empty() { + self.path.clone() + } else { + format!("{}?{}", self.path, self.query) + }; + let mut head = format!("{} {target} HTTP/1.1\r\nHost: {host}\r\n", self.method); + for (n, v) in &self.headers { + head.push_str(&format!("{n}: {v}\r\n")); + } + head.push_str(&format!("Content-Length: {}\r\n\r\n", self.body.len())); + head + } +} + +/// A response: its status, headers (names lowercase) and body. +#[derive(Debug)] +struct Resp { + status: u16, + headers: Vec<(String, String)>, + body: String, +} + +impl Resp { + fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find(|(n, _)| n == name) + .map(|(_, v)| v.as_str()) + } + + /// The status, the S3 headers every `/s3/` answer carries, and for an error its XML + /// `Code` and the request id the header names. + #[track_caller] + fn is(&self, status: u16, code: &str) -> &Self { + assert_eq!(self.status, status, "{self:?}"); + let id = self.header("x-amz-request-id").expect("a request id"); + assert!( + id.len() == 16 && id.bytes().all(|b| b.is_ascii_hexdigit()), + "{id}" + ); + assert_eq!(self.header("x-content-type-options"), Some("nosniff")); + assert_eq!(self.header("content-type"), Some("application/xml")); + assert!( + self.body + .starts_with(""), + "{}", + self.body + ); + assert!( + self.body.contains(&format!("{code}")), + "{}", + self.body + ); + assert!( + self.body + .contains(&format!("{id}")), + "{}", + self.body + ); + self + } +} + +/// A kept-alive connection to the server. +struct Conn { + host: String, + read: BufReader, + write: Option, +} + +impl Conn { + async fn open(host: &str) -> Self { + let sock = tokio::net::TcpStream::connect(host).await.unwrap(); + let (read, write) = sock.into_split(); + Conn { + host: host.to_string(), + read: BufReader::new(read), + write: Some(write), + } + } + + async fn send(&mut self, req: &Req) -> Resp { + let write = self.write.as_mut().expect("the connection's writing half"); + write + .write_all(req.head(&self.host).as_bytes()) + .await + .unwrap(); + write.write_all(&req.body).await.unwrap(); + self.response().await + } + + async fn response(&mut self) -> Resp { + let mut line = String::new(); + self.read.read_line(&mut line).await.unwrap(); + let status = line + .split_whitespace() + .nth(1) + .and_then(|s| s.parse().ok()) + .unwrap_or_else(|| panic!("no status in {line:?}")); + let mut headers = Vec::new(); + loop { + line.clear(); + self.read.read_line(&mut line).await.unwrap(); + let l = line.trim_end(); + if l.is_empty() { + break; + } + let (n, v) = l.split_once(':').unwrap(); + headers.push((n.to_ascii_lowercase(), v.trim().to_string())); + } + let len: usize = headers + .iter() + .find(|(n, _)| n == "content-length") + .map_or(0, |(_, v)| v.parse().unwrap()); + let mut body = vec![0; len]; + self.read.read_exact(&mut body).await.unwrap(); + Resp { + status, + headers, + body: String::from_utf8(body).unwrap(), + } + } + + /// Whether the server closed the connection: end of stream, or a reset. + async fn closed(&mut self) -> bool { + let mut byte = [0u8; 1]; + match tokio::time::timeout(Duration::from_secs(10), self.read.read(&mut byte)) + .await + .expect("the server closes or answers") + { + Ok(n) => n == 0, + Err(_) => true, + } + } +} + +/// One request on a connection of its own. +async fn send(host: &str, req: Req) -> Resp { + Conn::open(host).await.send(&req).await +} + +/// Every SigV4 refusal a server with credentials answers before knowing the caller, with +/// the key it takes as `good`: each one's answer, over a mode's `host`. +async fn sigv4_refusals(host: &str, good: (&str, &str)) { + let get = || Req::new("GET", "/s3/b/k"); + // An id it does not know, whose signature is consistent with it. + send(host, get().signed(host, ("AKIDNOTAKEY", "AKIDNOTAKEY"), 0)) + .await + .is(403, "InvalidAccessKeyId"); + // A good id, signed with another secret. + let resp = send(host, get().signed(host, (good.0, "not-the-secret"), 0)).await; + resp.is(403, "SignatureDoesNotMatch"); + assert!( + resp.body.contains("GET\n/s3/b/k\n"), + "{}", + resp.body + ); + // A clock 15 minutes and a second off, either way. + for skew in [-15 * 60 - 1, 15 * 60 + 1] { + let resp = send(host, get().signed(host, good, skew)).await; + resp.is(403, "RequestTimeTooSkewed"); + assert!(resp.body.contains(""), "{}", resp.body); + } + // Presigned, and temporary credentials. + send( + host, + get() + .query("X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=00") + .signed(host, good, 0), + ) + .await + .is(400, "AuthorizationQueryParametersError"); + send( + host, + get() + .header("x-amz-security-token", "t") + .signed(host, good, 0), + ) + .await + .is(400, "InvalidToken"); + // A signature that does not bind the payload. + let at = stamp(vk_peer::now_secs()); + let auth = authorization( + "GET", + "/s3/b/k", + "", + &[("host", host), ("x-amz-date", &at)], + UNSIGNED, + good, + &at, + ); + send( + host, + get() + .header("x-amz-content-sha256", UNSIGNED) + .header("x-amz-date", &at) + .header("Authorization", &auth), + ) + .await + .is(400, "AuthorizationHeaderMalformed"); + // And the good key passes the gate, to find nothing yet. + send(host, get().signed(host, good, 0)) + .await + .is(404, "NoSuchKey"); +} + +#[tokio::test] +async fn an_open_server_checks_no_signature_but_refuses_presigned_and_tokens() { + let dir = tmp("open"); + let host = spawn(open_state(&dir, true)); + let get = |path: &str| Req::new("GET", path); + send(&host, get("/s3/b/k")).await.is(404, "NoSuchKey"); + send(&host, get("/s3/b/k/deeper")) + .await + .is(404, "NoSuchKey"); + send(&host, get("/s3/b")).await.is(404, "NoSuchBucket"); + send(&host, get("/s3/b/")).await.is(404, "NoSuchBucket"); + send(&host, get("/s3/")).await.is(501, "NotImplemented"); + send(&host, get("/s3")).await.is(501, "NotImplemented"); + send( + &host, + Req::new("PUT", "/s3/b/k").body(b"Hello, World!".to_vec()), + ) + .await + .is(501, "NotImplemented"); + // Any key will do, its signature unchecked. + send( + &host, + get("/s3/b/k").signed(&host, ("anyone", "anything"), 0), + ) + .await + .is(404, "NoSuchKey"); + // The clock still is, and what is refused everywhere is refused here. + send( + &host, + get("/s3/b/k").signed(&host, ("anyone", "anything"), 3600), + ) + .await + .is(403, "RequestTimeTooSkewed"); + send(&host, get("/s3/b/k").query("X-Amz-Credential=a")) + .await + .is(400, "AuthorizationQueryParametersError"); + send(&host, get("/s3/b/k").header("X-Amz-Security-Token", "t")) + .await + .is(400, "InvalidToken"); +} + +#[tokio::test] +async fn a_shared_bearer_is_the_access_key_and_its_secret() { + let dir = tmp("bearer"); + let (state, signer) = shared( + &dir, + Auth::Bearer { + token: "s3cret".into(), + }, + true, + ); + let host = spawn(state); + let get = || Req::new("GET", "/s3/b/k"); + // No credential, or a refused one: S3's 403, not the 401 challenge. + let resp = send(&host, get()).await; + resp.is(403, "AccessDenied"); + assert_eq!(resp.header("www-authenticate"), None); + send(&host, get().header("Authorization", "Bearer nope")) + .await + .is(403, "AccessDenied"); + send(&host, get().header("Authorization", "AWS AKID:sig")) + .await + .is(403, "AccessDenied"); + // The bearer itself still works, as on `/dav/`. + send(&host, get().header("Authorization", "Bearer s3cret")) + .await + .is(404, "NoSuchKey"); + sigv4_refusals(&host, ("s3cret", "s3cret")).await; + + // A hub token is a key too, signed or as a bearer; an expired one, or one for another + // registry, is not. + let now = vk_peer::now_secs(); + let token = signer.sign(hub_claims(AUD, now - 1, now + 600)).unwrap(); + send(&host, get().signed(&host, (&token, &token), 0)) + .await + .is(404, "NoSuchKey"); + send( + &host, + get().header("Authorization", &format!("Bearer {token}")), + ) + .await + .is(404, "NoSuchKey"); + send(&host, get().signed(&host, (&token, "other"), 0)) + .await + .is(403, "SignatureDoesNotMatch"); + for bad in [ + signer + .sign(hub_claims(AUD, now - 7200, now - 3600)) + .unwrap(), + signer + .sign(hub_claims( + "0123456789abcdef0123456789abcdef", + now - 1, + now + 600, + )) + .unwrap(), + ] { + send(&host, get().signed(&host, (&bad, &bad), 0)) + .await + .is(403, "InvalidAccessKeyId"); + send( + &host, + get().header("Authorization", &format!("Bearer {bad}")), + ) + .await + .is(403, "AccessDenied"); + } +} + +#[tokio::test] +async fn the_shared_basic_user_is_the_access_key_and_the_password_its_secret() { + let dir = tmp("basic"); + let (state, _) = shared( + &dir, + Auth::Basic { + user: "ci".into(), + pass: "pa55".into(), + }, + true, + ); + let host = spawn(state); + let get = || Req::new("GET", "/s3/b/k"); + send(&host, get()).await.is(403, "AccessDenied"); + sigv4_refusals(&host, ("ci", "pa55")).await; + // The password is no access key, alone or as its own secret. + send(&host, get().signed(&host, ("pa55", "pa55"), 0)) + .await + .is(403, "InvalidAccessKeyId"); + // Basic credentials still work. + send(&host, get().header("Authorization", "Basic Y2k6cGE1NQ==")) + .await + .is(404, "NoSuchKey"); + send(&host, get().header("Authorization", "Basic Y2k6bm9wZQ==")) + .await + .is(403, "AccessDenied"); +} + +#[tokio::test] +async fn an_api_key_is_the_access_key_and_its_secret_until_revoked_or_expired() { + let dir = tmp("accounts"); + let state = accounts_state(&dir); + let db = accounts_db(&state); + let user = db + .upsert_user("https://issuer", "ci", EmailUpdate::Keep, None) + .unwrap(); + let scopes = [Scope { + action: Action::Read, + repo_pattern: "files/*".to_string(), + }]; + let (_, key) = db + .create_api_key(Some(&user.id), "s3", &scopes, None) + .unwrap(); + let (revoked, revoked_key) = db + .create_api_key(Some(&user.id), "gone", &scopes, None) + .unwrap(); + assert!(db.revoke_api_key(&user.id, &revoked.id).unwrap()); + let past = SystemTime::now() - Duration::from_secs(60); + let (_, expired_key) = db + .create_api_key(Some(&user.id), "old", &scopes, Some(past)) + .unwrap(); + let host = spawn(state.clone()); + let get = || Req::new("GET", "/s3/b/k"); + + send(&host, get()).await.is(403, "AccessDenied"); + send(&host, get().header("Authorization", "Bearer vkr_nope")) + .await + .is(403, "AccessDenied"); + send( + &host, + get().header("Authorization", &format!("Bearer {key}")), + ) + .await + .is(404, "NoSuchKey"); + sigv4_refusals(&host, (&key, &key)).await; + send(&host, get().signed(&host, ("vkr_0123", "vkr_0123"), 0)) + .await + .is(403, "InvalidAccessKeyId"); + for gone in [&revoked_key, &expired_key] { + send(&host, get().signed(&host, (gone, gone), 0)) + .await + .is(403, "InvalidAccessKeyId"); + send( + &host, + get().header("Authorization", &format!("Bearer {gone}")), + ) + .await + .is(403, "AccessDenied"); + } +} + +/// A refusal before authentication reads up to 64 KiB of the body, so a write sees its answer +/// rather than a reset, a large one while it is still sending; the connection then closes, +/// as after any refusal of a request with a body, rather than the server reading the rest. +/// Without a body, the connection goes on. +#[tokio::test] +async fn a_refused_body_is_read_up_to_64_kib() { + let dir = tmp("drain"); + let (state, _) = shared( + &dir, + Auth::Bearer { + token: "s3cret".into(), + }, + true, + ); + let host = spawn(state); + let put = |body: Vec| Req::new("PUT", "/s3/b/k").body(body); + type Refusal<'a> = (Box) -> Req + 'a>, u16, &'a str); + let refusals: Vec> = vec![ + (Box::new(put), 403, "AccessDenied"), + ( + Box::new(|b| put(b).header("Authorization", "Bearer nope")), + 403, + "AccessDenied", + ), + ( + Box::new(|b| put(b).signed(&host, ("AKIDNOTAKEY", "AKIDNOTAKEY"), 0)), + 403, + "InvalidAccessKeyId", + ), + ( + Box::new(|b| put(b).signed(&host, ("s3cret", "wrong"), 0)), + 403, + "SignatureDoesNotMatch", + ), + ( + Box::new(|b| put(b).signed(&host, ("s3cret", "s3cret"), 3600)), + 403, + "RequestTimeTooSkewed", + ), + ( + Box::new(|b| put(b).query("X-Amz-Signature=00")), + 400, + "AuthorizationQueryParametersError", + ), + ( + Box::new(|b| put(b).header("x-amz-security-token", "t")), + 400, + "InvalidToken", + ), + ]; + for (req, status, code) in &refusals { + let mut conn = Conn::open(&host).await; + let resp = conn.send(&req(Vec::new())).await; + resp.is(*status, code); + assert_eq!(resp.header("connection"), None, "{code}"); + conn.send(&Req::new("GET", "/s3/b/k").signed(&host, ("s3cret", "s3cret"), 0)) + .await + .is(404, "NoSuchKey"); + + // sccache's probe: 13 bytes, answered. + let mut conn = Conn::open(&host).await; + let resp = conn.send(&req(b"Hello, World!".to_vec())).await; + resp.is(*status, code); + assert_eq!(resp.header("connection"), Some("close"), "{code}"); + assert!(conn.closed().await, "{code}"); + + // A MiB of a body declared 64 MiB long, the rest never sent: the answer comes all + // the same, so the server did not wait for the body's end. + let req = req(vec![b'x'; 1 << 20]); + let mut conn = Conn::open(&host).await; + let head = req.head(&host).replace( + &format!("Content-Length: {}", 1 << 20), + &format!("Content-Length: {}", 64 << 20), + ); + let mut write = conn.write.take().unwrap(); + let writer = tokio::spawn(async move { + // The server stops reading: the write may fail once it closes. The writing half + // is kept open until then. + let _ = write.write_all(head.as_bytes()).await; + let _ = write.write_all(&req.body).await; + write + }); + let resp = tokio::time::timeout(Duration::from_secs(10), conn.response()) + .await + .expect("answered before the body ends"); + resp.is(*status, code); + assert_eq!(resp.header("connection"), Some("close"), "{code}"); + assert!(conn.closed().await, "{code}"); + drop(writer.await.unwrap()); + } +} + +/// `s3 = false`: `/s3/` is the 404 any unknown path is, after the usual gate. +#[tokio::test] +async fn s3_off_is_the_catch_all_404_after_authentication() { + let dir = tmp("off"); + let host = spawn(open_state(&dir, false)); + let off = send(&host, Req::new("GET", "/s3/b/k")).await; + let unknown = send(&host, Req::new("GET", "/nowhere")).await; + assert_eq!(off.status, 404); + assert_eq!(off.header("x-amz-request-id"), None); + assert_eq!(off.body, unknown.body); + assert_eq!(off.header("content-type"), unknown.header("content-type")); + + let dir = tmp("off-bearer"); + let (state, _) = shared( + &dir, + Auth::Bearer { + token: "s3cret".into(), + }, + false, + ); + let host = spawn(state); + let resp = send(&host, Req::new("GET", "/s3/b/k")).await; + assert_eq!(resp.status, 401, "the mode's own challenge"); + let resp = send( + &host, + Req::new("GET", "/s3/b/k").header("Authorization", "Bearer s3cret"), + ) + .await; + assert_eq!(resp.status, 404); + assert_eq!(resp.body, unknown.body); +} + +/// A registry its hub made read-only refuses S3 writes in S3's form, and reads go on. +#[tokio::test] +async fn a_read_only_registry_refuses_writes_with_retry_after() { + let dir = tmp("ro"); + let state = open_state(&dir, true); + state.store.set_read_only(true); + let host = spawn(state); + for method in ["PUT", "DELETE", "POST"] { + let resp = send(&host, Req::new(method, "/s3/b/k").body(b"x".to_vec())).await; + resp.is(503, "ServiceUnavailable"); + assert_eq!(resp.header("retry-after"), Some("60")); + } + send(&host, Req::new("GET", "/s3/b/k")) + .await + .is(404, "NoSuchKey"); +} From ca2301f7d28d351f0a82497a09410ff676368cd3 Mon Sep 17 00:00:00 2001 From: Antoine Bernardeau Date: Thu, 5 Nov 2026 10:55:46 +0100 Subject: [PATCH 04/54] vk-registry/files: let a write carry annotations and a precondition A front over the object layer can now see a body's bytes as they stream (for digests of its own), have annotations written on the object's manifest, and have a precondition on what is at the path checked under the namespace and store locks the tag is written under, so a create-only or compare-and-swap write cannot race another. WebDAV asks for none of it; its behaviour is unchanged. --- vk-registry/src/dav/files.rs | 17 ++++- vk-registry/src/files.rs | 136 ++++++++++++++++++++++++++++++----- vk-registry/src/lib.rs | 5 +- 3 files changed, 136 insertions(+), 22 deletions(-) diff --git a/vk-registry/src/dav/files.rs b/vk-registry/src/dav/files.rs index 021ccc74..5c370003 100644 --- a/vk-registry/src/dav/files.rs +++ b/vk-registry/src/dav/files.rs @@ -332,7 +332,7 @@ async fn put( } // Stream uploads to bound memory use; the digest falls out of the same pass. - let staged = match crate::files::stage(&state.store, req.into_body()).await { + let staged = match crate::files::stage(&state.store, req.into_body(), &mut |_| {}).await { Ok(staged) => staged, Err(PutError::TooLarge) => { return Ok(error_response( @@ -344,10 +344,14 @@ async fn put( Err(PutError::Failed(e)) => return Err(e), }; Ok( - match crate::files::commit(Arc::clone(&state.store), segs, staged).await? { + match crate::files::commit(Arc::clone(&state.store), segs, staged, Default::default()) + .await? + { Written::Created => created(), Written::Replaced => no_content(), Written::Refused(why) => put_refusal(why, href), + // Unreachable: no precondition was asked for. + Written::PreconditionFailed => no_content(), }, ) } @@ -355,7 +359,14 @@ async fn put( /// Read a request's body through and discard it: a status sent with request bytes unread /// resets the connection. `Some(413)` if the body crosses the object cap on the way. pub(super) async fn drain(req: Request, href: &str) -> Result>> { - match drain_into(&mut tokio::io::sink(), req.into_body(), MAX_OBJECT).await { + match drain_into( + &mut tokio::io::sink(), + req.into_body(), + MAX_OBJECT, + &mut |_| {}, + ) + .await + { Ok(_) => Ok(None), Err(PutError::TooLarge) => Ok(Some(error_response( StatusCode::PAYLOAD_TOO_LARGE, diff --git a/vk-registry/src/files.rs b/vk-registry/src/files.rs index 8fd1af22..2dd729e5 100644 --- a/vk-registry/src/files.rs +++ b/vk-registry/src/files.rs @@ -7,6 +7,7 @@ //! today, an S3 view in later work. Entries expire under gc's tag retention and blob grace //! windows. +use std::collections::BTreeMap; use std::path::PathBuf; use std::sync::{Arc, Mutex, PoisonError}; use std::time::{Duration, SystemTime}; @@ -227,8 +228,13 @@ pub(crate) struct Staged { } /// Stream `body` into a staging file under `uploads/`, hashing it on the way, up to -/// [`MAX_OBJECT`]. Memory use is bounded by the coalescing buffer whatever the size. -pub(crate) async fn stage(store: &Store, body: B) -> std::result::Result +/// [`MAX_OBJECT`]; `tap` sees every byte as it goes, for a front's own digests. Memory use is +/// bounded by the coalescing buffer whatever the size. +pub(crate) async fn stage( + store: &Store, + body: B, + tap: &mut impl FnMut(&[u8]), +) -> std::result::Result where B: hyper::body::Body + Unpin, B::Error: std::error::Error + Send + Sync + 'static, @@ -236,7 +242,7 @@ where let (path, file) = store.stage_file().map_err(PutError::Failed)?; let staging = Staging { path, armed: true }; let mut file = tokio::fs::File::from_std(file); - let written = drain_into(&mut file, body, MAX_OBJECT).await; + let written = drain_into(&mut file, body, MAX_OBJECT, tap).await; let flushed = file .flush() .await @@ -255,16 +261,34 @@ pub(crate) enum Written { /// the path changed while the body streamed and no longer takes an object; nothing was /// written Refused(Unwritable), + /// [`Commit::precondition`] refused the object found there, or its absence; nothing was + /// written + PreconditionFailed, +} + +/// Whether a [`commit`] goes ahead, given the object at its path or `None`. +pub(crate) type Precondition = Box) -> bool + Send>; + +/// What a front asks of a [`commit`] beyond the bytes. +#[derive(Default)] +pub(crate) struct Commit { + /// written on the object's manifest + pub(crate) annotations: BTreeMap, + /// checked under the locks the tag is written under, so no other write of this server + /// comes between the check and the write + pub(crate) precondition: Option, } /// Promote a [`stage`]d body into the pool and tag it at `segs` as a raw file, re-checking -/// [`unwritable`] under [`NAMESPACE`] and the shared store lock first. Off the runtime: -/// staging compresses, and a large object would otherwise block a tokio worker for seconds. -/// The locks are taken inside, after that pass — the store lock as the relay takes it. +/// [`unwritable`], then `how`'s precondition, under [`NAMESPACE`] and the shared store lock +/// first. Off the runtime: staging compresses, and a large object would otherwise block a +/// tokio worker for seconds. The locks are taken inside, after that pass — the store lock as +/// the relay takes it. pub(crate) async fn commit( store: Arc, segs: Vec, staged: Staged, + how: Commit, ) -> Result { tokio::task::spawn_blocking(move || -> Result { let Staged { @@ -278,11 +302,14 @@ pub(crate) async fn commit( // references it; see Store::lock_shared. Another write may have changed the path // while the body streamed, so it is checked again. let checked = store.lock_shared().and_then(|lock| { - let existed = matches!(resolve(&store, &segs)?, Some(Node::Object(_))); - Ok((lock, unwritable(&store, &segs)?, existed)) + let existing = match resolve(&store, &segs)? { + Some(Node::Object(obj)) => Some(obj), + _ => None, + }; + Ok((lock, unwritable(&store, &segs)?, existing)) }); - let (_lock, existed) = match checked { - Ok((lock, None, existed)) => (lock, existed), + let (_lock, existing) = match checked { + Ok((lock, None, existing)) => (lock, existing), Ok((_, Some(why), _)) => { promotion.discard(); return Ok(Written::Refused(why)); @@ -292,10 +319,17 @@ pub(crate) async fn commit( return Err(e); } }; + if let Some(precondition) = how.precondition + && !precondition(existing.as_ref()) + { + promotion.discard(); + return Ok(Written::PreconditionFailed); + } + let existed = existing.is_some(); store.promote_staged(&hex, promotion)?; staging.armed = false; let (repo, tag) = object_of(&segs).context("a write past its checks names an object")?; - store.put_raw_file(&repo, tag, &hex, size, None)?; + store.put_raw_file_annotated(&repo, tag, &hex, size, None, &how.annotations)?; Ok(if existed { Written::Replaced } else { @@ -307,12 +341,14 @@ pub(crate) async fn commit( } /// Stream `body` to `out`, coalescing small frames and enforcing `cap`, including for chunked -/// bodies; `(bytes written, sha256 hex)` on success. `out` may be a sink when draining a -/// refused request. The generic body allows tests with a small cap. +/// bodies; `(bytes written, sha256 hex)` on success. `tap` sees each frame's bytes as they are +/// hashed, before they are written. `out` may be a sink when draining a refused request. The +/// generic body allows tests with a small cap. pub(crate) async fn drain_into( out: &mut (impl AsyncWrite + Unpin), mut body: B, cap: u64, + tap: &mut impl FnMut(&[u8]), ) -> std::result::Result<(u64, String), PutError> where B: hyper::body::Body + Unpin, @@ -334,6 +370,7 @@ where return Err(PutError::TooLarge); } hasher.update(&data); + tap(&data); // A frame already worth a write goes straight out rather than through the // coalescing buffer, which is there for the small ones. if buf.is_empty() && data.len() >= STREAM_CHUNK { @@ -363,7 +400,7 @@ where B::Error: std::error::Error + Send + Sync + 'static, { // Past `cap`, or unreadable: either way there is nothing more to read. - let _ = drain_into(&mut tokio::io::sink(), body, cap).await; + let _ = drain_into(&mut tokio::io::sink(), body, cap, &mut |_| {}).await; } #[cfg(test)] @@ -410,7 +447,7 @@ mod tests { let mut file = tokio::fs::File::create(&path).await.unwrap(); assert!(matches!( - drain_into(&mut file, frames(), 250).await, + drain_into(&mut file, frames(), 250, &mut |_| {}).await, Err(PutError::TooLarge) )); drop(file); @@ -421,16 +458,81 @@ mod tests { // Exactly at the cap is not over it, and the digest is the body's. let mut file = tokio::fs::File::create(&path).await.unwrap(); - let (n, hex) = drain_into(&mut file, frames(), 400).await.ok().unwrap(); + let mut tapped = 0; + let (n, hex) = drain_into(&mut file, frames(), 400, &mut |b| tapped += b.len()) + .await + .ok() + .unwrap(); file.flush().await.unwrap(); drop(file); - assert_eq!(n, 400); + assert_eq!((n, tapped), (400, 400)); assert_eq!(std::fs::metadata(&path).unwrap().len(), 400); assert_eq!(hex, crate::sha256_hex_raw(&vec![1u8; 400])); let _ = std::fs::remove_dir_all(&dir); } + /// A commit writes the annotations it is given on the manifest, and its precondition sees + /// what is at the path, under the locks: refused, nothing is tagged and the staging file + /// goes. + #[tokio::test] + async fn a_commit_carries_annotations_and_honours_its_precondition() { + let dir = std::env::temp_dir().join(format!("vk-reg-files-commit-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let store = Arc::new(Store::new(dir.clone()).unwrap()); + let body = |b: &'static [u8]| { + let frame: std::result::Result<_, std::io::Error> = + Ok(hyper::body::Frame::data(Bytes::from_static(b))); + http_body_util::StreamBody::new(futures::stream::iter([frame])) + }; + let segs = vec!["b".to_string(), "k".to_string()]; + let write = async |b: &'static [u8], how: Commit| { + let staged = stage(&store, body(b), &mut |_| {}).await.ok().unwrap(); + commit(Arc::clone(&store), segs.clone(), staged, how) + .await + .unwrap() + }; + let absent = + || -> Option { Some(Box::new(|o: Option<&Object>| o.is_none())) }; + let annotations = BTreeMap::from([("org.example.k".to_string(), "v".to_string())]); + + let how = Commit { + annotations: annotations.clone(), + precondition: absent(), + }; + assert!(matches!(write(b"one", how).await, Written::Created)); + let (hex, _) = store.tag_target("files/b", "k").unwrap(); + let manifest: serde_json::Value = + serde_json::from_slice(&store.get_blob(&hex).unwrap().unwrap()).unwrap(); + assert_eq!(manifest["annotations"]["org.example.k"], "v"); + assert!(crate::raw_file_layer(&serde_json::to_vec(&manifest).unwrap()).is_some()); + + let how = Commit { + precondition: absent(), + ..Commit::default() + }; + assert!(matches!( + write(b"two", how).await, + Written::PreconditionFailed + )); + assert_eq!(store.tag_target("files/b", "k").unwrap().0, hex); + let staged = std::fs::read_dir(dir.join("uploads")) + .unwrap() + .filter(|e| e.as_ref().unwrap().file_type().unwrap().is_file()) + .count(); + assert_eq!(staged, 0, "the refused body's staging file is gone"); + + let layer = crate::sha256_hex_raw(b"one"); + let how = Commit { + precondition: Some(Box::new(move |o: Option<&Object>| { + o.is_some_and(|o| o.layer == layer && o.size == 3) + })), + ..Commit::default() + }; + assert!(matches!(write(b"three", how).await, Written::Replaced)); + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn paths_map_onto_repositories_and_tags() { let segs = |s: &[&str]| s.iter().map(|s| s.to_string()).collect::>(); diff --git a/vk-registry/src/lib.rs b/vk-registry/src/lib.rs index 98aed77e..021dbcb7 100644 --- a/vk-registry/src/lib.rs +++ b/vk-registry/src/lib.rs @@ -746,8 +746,9 @@ impl Store { self.put_raw_file_annotated(name, tag, layer_hex, size, title, &BTreeMap::new()) } - /// [`Store::put_raw_file`] with `annotations` on the manifest: a [`Retention`]. - fn put_raw_file_annotated( + /// [`Store::put_raw_file`] with `annotations` on the manifest: a [`Retention`], or what + /// a `files/` front keeps beside an object. + pub(crate) fn put_raw_file_annotated( &self, name: &str, tag: &str, From 964fcadab775f627820a0f8b1fa57a5d6403e8bd Mon Sep 17 00:00:00 2001 From: Antoine Bernardeau Date: Thu, 5 Nov 2026 11:06:17 +0100 Subject: [PATCH 05/54] vk-registry/s3: serve, store and delete objects Serve the plain-file area's objects under /s3/: PutObject, GetObject, HeadObject, GetObjectAttributes and DeleteObject, with HeadBucket, GetBucketLocation and ListBuckets, over the object layer WebDAV uses and under its authorization and disclosure rules. A key maps onto the store's names as they are; anything else is refused naming the rule. A write records the object's MD5 (its ETag) and user metadata on the manifest, checks the signed payload digest and the checksums it is sent, and checks If-None-Match/If-Match under the namespace lock; reads take the four conditional headers and the checksum mode. No content type is kept. Listings, aws-chunked bodies, ranges, copies and multipart answer NotImplemented until they land. --- CHANGELOG.md | 12 + Cargo.lock | 3 + Cargo.toml | 9 +- README.md | 18 +- vk-registry/Cargo.toml | 5 + vk-registry/DESIGN.md | 112 +++- vk-registry/src/dav/mod.rs | 2 +- vk-registry/src/files.rs | 11 +- vk-registry/src/lib.rs | 20 + vk-registry/src/s3/bucket.rs | 128 +++++ vk-registry/src/s3/conditional.rs | 324 ++++++++++++ vk-registry/src/s3/meta.rs | 160 ++++++ vk-registry/src/s3/mod.rs | 200 ++++++-- vk-registry/src/s3/object.rs | 654 ++++++++++++++++++++++++ vk-registry/src/s3/path.rs | 259 ++++++++++ vk-registry/src/s3/xml.rs | 62 ++- vk-registry/tests/s3_e2e.rs | 813 +++++++++++++++++++++++++++++- 17 files changed, 2701 insertions(+), 91 deletions(-) create mode 100644 vk-registry/src/s3/bucket.rs create mode 100644 vk-registry/src/s3/conditional.rs create mode 100644 vk-registry/src/s3/meta.rs create mode 100644 vk-registry/src/s3/object.rs create mode 100644 vk-registry/src/s3/path.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index c7be659b..cdc633e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ All notable changes to virtkit will be documented in this file. ## [Unreleased] +### Added + +- **The registry's plain-file area is also served as S3.** Under `/s3/` on `vk-registry`, + each top-level directory of the WebDAV `files/` area is a bucket, so `sccache`'s S3 backend + and other S3 clients can share a compiler cache or files with it: point them at + `https:///s3` with any region, and use the registry credential (an API key, a hub + token or the shared bearer) as both the access key and the secret access key, or the shared + Basic user and its password. Objects can be written, read with conditional headers and + checksums, given metadata and deleted, and are the same files WebDAV and `vk registry get` + see. Listing objects, multipart and chunked uploads, ranges and copies are not served yet. + Set `s3 = false` in the server config to turn it off. + ## [0.106.0] - 2026-11-05 ### Added diff --git a/Cargo.lock b/Cargo.lock index 096ef156..9cd85fb4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4304,6 +4304,7 @@ dependencies = [ "anyhow", "bytes", "clap", + "crc32fast", "futures", "hmac", "http-body-util", @@ -4312,6 +4313,7 @@ dependencies = [ "hyper-util", "jemallocator", "libc", + "md5", "oci-client", "rand", "redb", @@ -4319,6 +4321,7 @@ dependencies = [ "rustls", "serde", "serde_json", + "sha1", "sha2", "tokio", "tokio-rustls", diff --git a/Cargo.toml b/Cargo.toml index 8124b74f..d1a8bce5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -128,9 +128,14 @@ oci-client = { version = "0.18", default-features = false } flate2 = "1" # docker-hash: content hashing of a (Dockerfile, stage) for image-tag resolution sha2 = "0.11" -# vk-tpm's SHA-1 PCR bank (TPM 2.0 keeps one for compatibility). RustCrypto, like sha2; already -# in the tree via russh. +# vk-tpm's SHA-1 PCR bank (TPM 2.0 keeps one for compatibility), and vk-registry's check of an +# S3 upload's x-amz-checksum-sha1. RustCrypto, like sha2; already in the tree via russh. sha1 = "0.11" +# vk-registry: the MD5 an S3 object's ETag is, and the check of a Content-MD5. Already in the +# tree via russh. +md5 = "0.8" +# vk-registry: the check of an S3 upload's x-amz-checksum-crc32. Already in the tree via flate2. +crc32fast = "1.5" # vk-tpm: the TPM's entropy (TPM2_GetRandom, its seeds) from the OS. Already in the tree via rand. getrandom = "0.4" # vk-tpm: constant-time comparison of authorization values. Already in the tree via the diff --git a/README.md b/README.md index fe7872ab..58636f40 100644 --- a/README.md +++ b/README.md @@ -855,6 +855,22 @@ with `read:files//*` only reads it. Cached files expire under `vk-registry retention like any other tag, unless their manifest keeps or expires them. Set `webdav = false` in the server config to disable WebDAV. +The same plain-file area is served as S3 under `/s3/`, a bucket being a top-level directory +of `/dav/files/`, so `sccache`'s S3 backend, the AWS CLI and SDKs and other S3 clients can use +it. The access key is the registry credential — an API key, a hub token, or the shared bearer — +and the secret access key is the same string; with a shared Basic user, the user name and its +password. For `sccache`: + +```bash +SCCACHE_BUCKET=sccache SCCACHE_ENDPOINT=https:///s3 SCCACHE_REGION=auto \ +SCCACHE_S3_KEY_PREFIX= AWS_ACCESS_KEY_ID=$KEY AWS_SECRET_ACCESS_KEY=$KEY sccache … +``` + +Object keys follow the store's names (`A-Z a-z 0-9 . _ -` between slashes, at most 1024 +bytes); objects are at most 4 GiB; presigned URLs are refused. Listing a bucket's objects, +multipart and chunked uploads, ranges and copies are not served yet. Set `s3 = false` to +disable it. + `vk registry put [--dir ] [--ttl 7d | --keep] ...` stores files there, each once by digest (`/dav/files/sha256/`) and, with `--dir`, by name (`/dav/files//`), and `vk registry get ` fetches one back, checked against its digest; @@ -883,7 +899,7 @@ See [`docs/registry-hub.md`](docs/registry-hub.md). | --- | --- | | `vk` | Host CLI, VMM, image builder, userspace network, compose runner, and GitLab executor. It embeds the default guest kernel and `vk-agent`. | | `vk-agent` | Guest PID 1 and command server. It configures mounts, networking, hostname, shared directories, optional SSH, and host-driven execution over vsock. | -| `vk-registry` | Optional OCI-distribution server with a pull-through cache, a WebDAV view of the store with a plain-file area for compiler caches, and build-once locking. | +| `vk-registry` | Optional OCI-distribution server with a pull-through cache, a WebDAV view of the store with a plain-file area for compiler caches, also served as S3, and build-once locking. | | `vk-hub` | Experimental hub for a fleet of `vk node` hosts, and a web UI for the VMs on this machine: `vk-hub local` serves it on loopback. See the [fleet design](docs/fleet-design.md) and [prototype reference](docs/fleet-prototype.md). | | `vk-gitlab` | Experimental GitLab runner for a `vk-hub` fleet: takes jobs from GitLab and has the hub run each on a `vk node`. See [GitLab dispatch](docs/gitlab-dispatch.md). | | `vk-runnerctl` | Optional root-side helper that adjusts GitLab runner concurrency within an administrator-configured range. | diff --git a/vk-registry/Cargo.toml b/vk-registry/Cargo.toml index 894afe18..93908a47 100644 --- a/vk-registry/Cargo.toml +++ b/vk-registry/Cargo.toml @@ -35,6 +35,11 @@ rustls.workspace = true tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "logging", "tls12"] } oci-client.workspace = true sha2.workspace = true +# The digests S3 clients send or expect beside SHA-256: an object's MD5 (its ETag), and the +# SHA-1 and CRC32 an upload may be checked against. +md5.workspace = true +sha1.workspace = true +crc32fast.workspace = true # SigV4's HMAC chain for /s3/; zeroize wipes the keys it derives. hmac.workspace = true zeroize.workspace = true diff --git a/vk-registry/DESIGN.md b/vk-registry/DESIGN.md index 367422c4..6383bee2 100644 --- a/vk-registry/DESIGN.md +++ b/vk-registry/DESIGN.md @@ -558,6 +558,112 @@ an entry alive as a writer's do. There is no per-directory policy and no size ca replaced or deleted object frees no disk until the next `gc`. A staging file a crashed `PUT` left in `uploads/` goes with the idle uploads. +## S3 view (`/s3/`) + +The `files/` area is also served as S3, path-style, under `/s3/`, through the same listener, +TLS and credentials: a bucket is a top-level `files/` directory and an object key the path +below it, so `PUT /s3/sccache/a/b/abcdef` writes the same tag `/dav/files/sccache/a/b/abcdef` +names. Nothing new is stored, and the WebDAV view, `/v2/`, `/browse`, `vk registry get` and +the gc see the same entries. S3 is enabled by default. Set `s3 = false` to disable it: `/s3/` +requests then return the 404 any unknown path returns, after the usual authentication, and +the server logs this setting at startup. + +```text +/s3/ the buckets: top-level directories of files/ +/s3/ files/ +/s3// the object files// names, as in /dav/files/ +``` + +Clients take `endpoint = https:///s3` and any region: `sccache` with +`SCCACHE_BUCKET= SCCACHE_ENDPOINT=https:///s3 SCCACHE_REGION=auto` +(`SCCACHE_S3_KEY_PREFIX` names a directory inside the bucket). Virtual-hosted style, which +needs wildcard DNS, is not served. + +**Credentials.** SigV4 needs the server to hold the signing secret, and this registry stores +API keys only as hashes and never mints hub tokens, so no new secret is introduced: the +access key id is a credential the registry already takes, and the secret access key is the +same string. In accounts mode `AWS_ACCESS_KEY_ID=vkr_…` and `AWS_SECRET_ACCESS_KEY=vkr_…`, +the same value; a hub token likewise, in either mode, when the registry takes them; the +shared bearer likewise. With a shared Basic user, the access key id is the user name and the +secret its password, which then never crosses the wire; a user name holding `/` cannot be an +access key id, since the credential scope is split from the right. The access key id is +authenticated first, exactly as the bearer it would be, so revocation, expiry and scopes +apply unchanged, and only then is its secret used to check the signature; an unknown id is +403 `InvalidAccessKeyId`, a wrong signature 403 `SignatureDoesNotMatch` (with the string to +sign and canonical request, as AWS answers). The request's time must be on the scope's day +and within 15 minutes of the server's (403 `RequestTimeTooSkewed`, naming the server's +time). The usual `Authorization: Bearer|Basic` and the session cookie keep working on `/s3/`; +a request with no credential on a server that requires one is S3's 403 `AccessDenied`, not +the 401 challenge. On a server with no credential configured, signatures are not checked, as +nothing else is. Presigned (query-string) URLs are refused with 400 +`AuthorizationQueryParametersError`, since they put a credential in a URL and so in logs, and +temporary credentials (`x-amz-security-token`) with 400 `InvalidToken`. + +The exposure is today's: a bearer travels in clear under TLS and so does an access key id, +while the signature also binds the method, path, query, signed headers and a timestamp. With +`UNSIGNED-PAYLOAD`, which OpenDAL and most SDKs send, a captured `Authorization` header can be +replayed with another body to the same key for those 15 minutes, as a captured bearer can be +for as long as it is good. + +**Keys** follow the store's name rules as they are; there is no escaping scheme. A bucket is +one component; a key is 1 to 14 components (16 counting `files` and the bucket), each a run of +`[A-Za-z0-9._-]` of at most 255 bytes, not `.`, `..`, `tags`, `manifests` or `blobs`, the whole +key at most 1024 bytes, no empty component and no trailing slash, so there are no "directory +marker" objects. Paths are split before they are percent-decoded, as WebDAV's are, so `%2F` +cannot smuggle a separator. Anything else is 400 `InvalidArgument`, `KeyTooLongError` or +`InvalidBucketName`, with a message naming the rule. `sccache`'s keys (`a/b/c/`, +`.sccache_check`) pass. + +**Authorization** is the WebDAV view's: a `PUT` needs `Write` on the key's parent repository, +a read `Read` on the repository the key resolves to, and a `DELETE` `Write` there; a caller +holding the action on neither the key's own name nor its parent's is 403 `AccessDenied` before +anything is looked up, and what it may not read is otherwise absent (`NoSuchKey`). A write +refused over a clash answers 409 and so reveals that the name exists, as WebDAV's does. A +bucket authorizes as `files/`. Listing the buckets needs a configured credential, as +`/dav/`'s root listing does, and shows only those the caller may read. + +**Objects are bytes.** Every object is served `application/octet-stream`, +`Content-Disposition: attachment` and `nosniff`, whatever its writer sent: a `Content-Type` +is not stored, for the reason manifest media types are restricted (this origin also serves +`/browse` and holds the session cookie). An object's ETag is the MD5 of its bytes, quoted, +which a `PutObject` computes in the same pass as the SHA-256 and records on the manifest as +`org.virtkit.md5`; an object written otherwise (WebDAV, `vk registry put`, `/upload`) has no +MD5 and its ETag is its SHA-256, quoted — 64 hex digits, which a client checking an ETag as +an MD5 sees is not one. User metadata (`x-amz-meta-`, names `[a-z0-9-]`, at most 2 KiB +of names and values) is kept on the manifest as `org.virtkit.s3-meta.` and sent back on +reads; a replaced object keeps only what its own write sent. + +| Request | Answer | +|---|---| +| `PUT /s3//` PutObject | 200 with `ETag`; streamed to `uploads/` while hashed (SHA-256, MD5, and the SHA-1 or CRC32 when asked), then tagged under the namespace lock as a WebDAV `PUT` is. `x-amz-content-sha256` is `UNSIGNED-PAYLOAD` or the body's SHA-256, else 400 `XAmzContentSHA256Mismatch` and nothing is tagged. `Content-MD5` and `x-amz-checksum-{sha256,sha1,crc32}` are checked (400 `BadDigest`; 400 `InvalidDigest`/`InvalidRequest` when malformed) and the checksums echoed; `crc32c` and `crc64nvme` are taken unchecked. `If-None-Match: *` (create only) and `If-Match: ` (replace only that object; an absent one fails too) are checked under the lock the tag is written under: 412 `PreconditionFailed`. 409 `OperationAborted` onto a directory, below an object, or over a foreign tag; 413 `EntityTooLarge` past 4 GiB, declared or streamed; 400 `MetadataTooLarge` | +| `GET`, `HEAD /s3//` GetObject, HeadObject | 200 with the bytes, `ETag`, `Last-Modified` from the tag, `Content-Length` and the metadata; `x-amz-checksum-sha256` (the store's digest, base64) with `x-amz-checksum-mode: ENABLED`. `If-Match`, `If-Unmodified-Since`, `If-None-Match` and `If-Modified-Since` in RFC 9110's order: 412 `PreconditionFailed`, or 304 with the `ETag`. `Range` is ignored and no `Accept-Ranges` is sent: a whole 200 is a legal answer. A directory, a foreign tag or a missing key is 404 `NoSuchKey`. A `GET` refreshes an idle tag as WebDAV's does; a `HEAD` touches nothing | +| `GET /s3//?attributes` GetObjectAttributes | 200 `GetObjectAttributesResponse` with what `x-amz-object-attributes` names of `ETag`, `Checksum` (SHA-256), `StorageClass` (`STANDARD`) and `ObjectSize`, and `Last-Modified`; 400 `InvalidArgument` without the header; touches nothing | +| `DELETE /s3//` DeleteObject | 204 whether or not the key held an object, as S3 answers, and for a key that is a directory, which holds no object to delete; 409 `OperationAborted` for a foreign tag; 403 when the caller may read the object but not write it | +| `HEAD /s3/` HeadBucket | 200 when `files/` is a directory, 404 `NoSuchBucket` when not; 403 without `Read` on `files/` | +| `GET /s3/?location` GetBucketLocation | the region the request was signed for, which is any the client chose, or an empty `LocationConstraint` for an unsigned request; after HeadBucket's checks | +| `GET /s3/` ListBuckets | `ListAllMyBucketsResult`: the readable top-level directories, each dated by its directory's mtime, and the owner `vk-registry`; 403 `AccessDenied` on a server with no credential | +| anything else | 501 `NotImplemented`: listing a bucket's objects, creating and deleting buckets, copies, batch deletes, multipart uploads, `aws-chunked` bodies (`STREAMING-*` payloads, `Content-Encoding: aws-chunked`), tagging, ACLs, versioning and every other subresource; an SDK's `x-id` operation hint is ignored | + +**`LastModified` moves on reads**, as `getlastmodified` does over WebDAV: a `GET` refreshes +the tag's mtime once it is an hour old, which is what keeps a hot cache entry from the gc's +retention. Sync tools that compare modification times (`aws s3 sync`, rclone `--update`) see +it; they keep the source file's own time in user metadata. + +**Errors** are S3's XML `Error` documents (`Code`, `Message`, `Resource`, `RequestId`, and +the extra elements AWS sends, such as `ServerTime` or `Condition`), so a client shows a code +rather than a bare status. Every `/s3/` response carries an `x-amz-request-id` of 16 hex +digits and `nosniff`; an internal error logs its request id, so a client's report can be +matched to the server's line. A refusal decided before the caller is authenticated reads at +most 64 KiB of the body first, enough for `sccache`'s 13-byte probe to see its answer rather +than a reset; every refusal after it reads the body through, up to the 4 GiB cap, as +WebDAV's do. A refusal of a request with a body then closes the connection. A registry its +hub made read-only answers writes 503 `ServiceUnavailable` with `Retry-After`. + +S3 writes into `files/` take the WebDAV view's namespace lock, run their store work on the +blocking pool and stream their bodies on the runtime. Objects expire under the gc's tag +retention like every `files/` entry ("Lifecycle" above): a PutObject writes no retention +annotation. + ## Accounts administration `vk-registry accounts` manages users, sessions, administrators, and API keys. The command @@ -813,7 +919,9 @@ when that is still below `max_connections` plus 256. The unit `install-service` `/browse`, `status` and `gc`. - The `files/` area trusts its writers: a stored compiler-cache entry is linked into every project computing the same key, so write access belongs to trusted pipelines only (see - "WebDAV view"). + "WebDAV view"). A key reaches exactly the same content over `/v2/`, `/dav/` and `/s3/`. +- SigV4 needs the server's clock within 15 minutes of its clients'. A reverse proxy in front + of `/s3/` must pass `Host` and the path unchanged, since the signature covers both. - The store directory must be writable only by trusted local users. Requests can follow symlinks in parent directories under `repos/files/`; checks reject only a symlink at the final path component. Preventing this requires descriptor-relative path resolution. @@ -825,6 +933,6 @@ when that is still below `max_connections` plus 256. The unit `install-service` - The server has no durable request audit log. Administrative changes are logged to stderr: `accounts admin: :` identifies the admin socket peer; `accounts web: :` identifies the session for browse and settings changes (tag deletion, API key minting and - revocation, captions, uploads). Writes through `/v2/` and WebDAV, including tag removal + revocation, captions, uploads). Writes through `/v2/`, WebDAV and S3, including tag removal and replacement, are not audited. - OIDC trusts platform roots only and caches discovery until restart. diff --git a/vk-registry/src/dav/mod.rs b/vk-registry/src/dav/mod.rs index 21102d3d..06714308 100644 --- a/vk-registry/src/dav/mod.rs +++ b/vk-registry/src/dav/mod.rs @@ -228,7 +228,7 @@ pub(crate) fn href(segments: &[&str], collection: bool) -> String { /// Root enumeration requires configured credentials. Accounts-mode listings also filter by /// scope. -fn may_enumerate(state: &ServerState) -> bool { +pub(crate) fn may_enumerate(state: &ServerState) -> bool { match &state.auth { Authenticator::Accounts { .. } => true, Authenticator::Shared(auth) => auth.enabled(), diff --git a/vk-registry/src/files.rs b/vk-registry/src/files.rs index 2dd729e5..c1da7212 100644 --- a/vk-registry/src/files.rs +++ b/vk-registry/src/files.rs @@ -3,8 +3,8 @@ //! single-layer raw-file manifest, as written by `/upload`, whose layer holds the file bytes. //! A write streams to `uploads/` while hashing ([`stage`]), then promotes the blob and writes //! the manifest and tag under the namespace lock ([`commit`]). Each wire protocol over this -//! layer keeps its own status codes and bodies: WebDAV under `/dav/files/` (`crate::dav`) -//! today, an S3 view in later work. Entries expire under gc's tag retention and blob grace +//! layer keeps its own status codes and bodies: WebDAV under `/dav/files/` (`crate::dav`) and +//! S3 under `/s3/` (`crate::s3`). Entries expire under gc's tag retention and blob grace //! windows. use std::collections::BTreeMap; @@ -75,6 +75,8 @@ pub(crate) struct Object { pub(crate) size: u64, /// the tag's mtime pub(crate) modified: SystemTime, + /// the manifest's annotations: what a front stored beside the bytes + pub(crate) annotations: BTreeMap, } impl Node { @@ -118,16 +120,17 @@ pub(crate) fn resolve(store: &Store, segs: &[String]) -> Result> { return Ok(None); }; let layer = match store.get_blob(&manifest_hex)? { - Some(manifest) => crate::raw_file_layer(&manifest), + Some(manifest) => crate::raw_file_parts(&manifest), None => None, }; Ok(Some(match layer { - Some((layer, size)) => Node::Object(Object { + Some((layer, size, annotations)) => Node::Object(Object { repo, tag: tag.to_string(), layer, size, modified, + annotations, }), None => Node::Foreign(repo), })) diff --git a/vk-registry/src/lib.rs b/vk-registry/src/lib.rs index 021dbcb7..38bb187a 100644 --- a/vk-registry/src/lib.rs +++ b/vk-registry/src/lib.rs @@ -2935,7 +2935,27 @@ pub fn raw_file_title(manifest: &[u8]) -> Option { /// any other shape: an image pushed over `/v2/` into a `files/` repository is not a file /// the WebDAV view serves. pub fn raw_file_layer(manifest: &[u8]) -> Option<(String, u64)> { + raw_file_layer_of(&serde_json::from_slice(manifest).ok()?) +} + +/// [`raw_file_layer`] and the manifest's string annotations, from one parse; an annotation +/// whose value is not a string is left out. +pub(crate) fn raw_file_parts(manifest: &[u8]) -> Option<(String, u64, BTreeMap)> { let v: serde_json::Value = serde_json::from_slice(manifest).ok()?; + let (hex, size) = raw_file_layer_of(&v)?; + let annotations = v + .pointer("/annotations") + .and_then(|a| a.as_object()) + .map(|a| { + a.iter() + .filter_map(|(k, v)| Some((k.clone(), v.as_str()?.to_string()))) + .collect() + }) + .unwrap_or_default(); + Some((hex, size, annotations)) +} + +fn raw_file_layer_of(v: &serde_json::Value) -> Option<(String, u64)> { let [layer] = v.pointer("/layers")?.as_array()?.as_slice() else { return None; }; diff --git a/vk-registry/src/s3/bucket.rs b/vk-registry/src/s3/bucket.rs new file mode 100644 index 00000000..e6a87a77 --- /dev/null +++ b/vk-registry/src/s3/bucket.rs @@ -0,0 +1,128 @@ +//! The service and its buckets: ListBuckets, HeadBucket and GetBucketLocation. A bucket is a +//! top-level directory of `files/`, authorized as the repository `files/`; one the +//! caller may not read is absent to it, as in the WebDAV view. + +use hyper::body::Incoming; +use hyper::header::{AUTHORIZATION, CONTENT_LENGTH}; +use hyper::{Request, Response, StatusCode}; + +use super::{Ctx, refuse, sigv4, xml}; +use crate::files::{FILES_REPO, allowed}; +use crate::{Authz, Body, ServerState, accounts, body_of}; + +/// The owner every listing names: the registry, whoever wrote what it holds. +pub(super) const OWNER: &str = "vk-registry"; + +/// `` as S3's listings carry it. +pub(super) fn owner() -> String { + format!( + "{}{}", + xml::element("ID", OWNER), + xml::element("DisplayName", OWNER) + ) +} + +/// `GET /s3/`: the top-level directories the caller may read. Refused, as `/dav/`'s root +/// listing is, on a server without a credential, where it would list for anyone. +pub(super) async fn list( + state: &ServerState, + authz: &Authz<'_>, + req: Request, + ctx: &Ctx, +) -> anyhow::Result> { + if !crate::dav::may_enumerate(state) { + let resp = ctx.error( + StatusCode::FORBIDDEN, + "AccessDenied", + "listing the buckets needs a configured credential", + ); + return refuse(req, resp, ctx).await; + } + crate::dav::blocking(state, authz, move |store, authz| { + let mut buckets = String::new(); + for name in store.repo_children(FILES_REPO)? { + let repo = format!("{FILES_REPO}/{name}"); + if !authz.may_read(&repo) { + continue; + } + let created = store + .repos_path_modified(&repo) + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + buckets.push_str(&format!( + "{}{}", + xml::element("Name", &name), + xml::element("CreationDate", &sigv4::format_time(created)) + )); + } + Ok(xml::document( + "ListAllMyBucketsResult", + &format!("{}{buckets}", owner()), + )) + }) + .await +} + +/// Whether `bucket` is there for this caller: `Err` with the answer when it is not — 403 when +/// the caller may not read the bucket at all, 404 `NoSuchBucket` when it may but no such +/// directory exists. +async fn check( + state: &ServerState, + authz: &Authz<'_>, + bucket: &str, + ctx: &Ctx, +) -> anyhow::Result>> { + let repo = format!("{FILES_REPO}/{bucket}"); + if !allowed(authz, accounts::Action::Read, &repo) { + return Ok(Some(ctx.access_denied())); + } + let store = std::sync::Arc::clone(&state.store); + let exists = tokio::task::spawn_blocking(move || store.repo_dir_exists(&repo)).await?; + Ok((!exists).then(|| { + ctx.error( + StatusCode::NOT_FOUND, + "NoSuchBucket", + "the specified bucket does not exist", + ) + })) +} + +/// `HEAD /s3/`. +pub(super) async fn head( + state: &ServerState, + authz: &Authz<'_>, + bucket: String, + req: Request, + ctx: &Ctx, +) -> anyhow::Result> { + if let Some(resp) = check(state, authz, &bucket, ctx).await? { + return refuse(req, resp, ctx).await; + } + Ok(Response::builder() + .status(StatusCode::OK) + .header(CONTENT_LENGTH, "0") + .body(body_of(bytes::Bytes::new()))?) +} + +/// `GET /s3/?location`: the region the request was signed for, which is any the +/// client chose; empty when it was not signed. +pub(super) async fn location( + state: &ServerState, + authz: &Authz<'_>, + bucket: String, + req: Request, + ctx: &Ctx, +) -> anyhow::Result> { + if let Some(resp) = check(state, authz, &bucket, ctx).await? { + return refuse(req, resp, ctx).await; + } + // The gate parsed this header already; a request it let through without one parsing is + // one that was not signed. + let region = req + .headers() + .get(AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .and_then(|v| sigv4::parse_authorization(v).ok()) + .map_or(String::new(), |c| xml::escape(c.region)); + Ok(xml::document("LocationConstraint", ®ion)) +} diff --git a/vk-registry/src/s3/conditional.rs b/vk-registry/src/s3/conditional.rs new file mode 100644 index 00000000..1887178e --- /dev/null +++ b/vk-registry/src/s3/conditional.rs @@ -0,0 +1,324 @@ +//! ETags and the conditional headers S3 takes on them: on a read, `If-Match`, +//! `If-Unmodified-Since`, `If-None-Match` and `If-Modified-Since`, in RFC 9110's order (S3's +//! too); on a write, `If-None-Match: *` (create only) and `If-Match` (replace only that +//! object), checked where the tag is written. + +use std::collections::BTreeMap; +use std::time::{Duration, SystemTime}; + +use hyper::header::{HeaderMap, IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE}; + +use crate::files::{Object, Precondition}; + +/// The manifest annotation holding an S3-written object's MD5, as lowercase hex. +pub(crate) const MD5_ANNOTATION: &str = "org.virtkit.md5"; + +/// An object's ETag, quoted: the MD5 an S3 write recorded, or the SHA-256 of its bytes for +/// one written otherwise (WebDAV, `vk registry put`). 64 hex digits, so a client that checks +/// an ETag as an MD5 sees it is not one rather than a mismatch. +pub(crate) fn etag(layer: &str, annotations: &BTreeMap) -> String { + match annotations.get(MD5_ANNOTATION) { + Some(md5) + if md5.len() == 32 + && md5 + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) => + { + format!("\"{md5}\"") + } + _ => format!("\"{layer}\""), + } +} + +/// [`etag`] of an object. +pub(crate) fn etag_of(obj: &Object) -> String { + etag(&obj.layer, &obj.annotations) +} + +/// Whether an `If-Match`/`If-None-Match` list names `etag` (quoted): `*` names any, a weak +/// `W/` tag compares as its strong form, and an unquoted entry as if it were quoted. +pub(crate) fn names(list: &str, etag: &str) -> bool { + list.split(',').map(str::trim).any(|entry| { + let entry = entry.strip_prefix("W/").unwrap_or(entry); + entry == "*" + || entry == etag + || etag.strip_prefix('"').and_then(|e| e.strip_suffix('"')) == Some(entry) + }) +} + +/// A read's conditional headers. A date that does not parse is ignored, as RFC 9110 has it. +#[derive(Debug, Default)] +pub(crate) struct Conditions { + if_match: Option, + if_none_match: Option, + if_modified_since: Option, + if_unmodified_since: Option, +} + +/// What a read's [`Conditions`] decide. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Outcome { + Serve, + /// 304 + NotModified, + /// 412, naming the header that failed + Failed(&'static str), +} + +impl Conditions { + pub(crate) fn of(headers: &HeaderMap) -> Self { + let text = |name| { + let values: Vec<&str> = headers + .get_all(name) + .iter() + .filter_map(|v| v.to_str().ok()) + .collect(); + (!values.is_empty()).then(|| values.join(",")) + }; + let date = |name| { + headers + .get(name) + .and_then(|v| v.to_str().ok()) + .and_then(|v| httpdate::parse_http_date(v.trim()).ok()) + }; + Conditions { + if_match: text(IF_MATCH), + if_none_match: text(IF_NONE_MATCH), + if_modified_since: date(IF_MODIFIED_SINCE), + if_unmodified_since: date(IF_UNMODIFIED_SINCE), + } + } + + /// Decide on an object with this `etag` (quoted) last modified at `modified`. Times + /// compare to the second, the precision an HTTP date has. `If-Unmodified-Since` counts + /// only without `If-Match`, `If-Modified-Since` only without `If-None-Match`. + pub(crate) fn evaluate(&self, etag: &str, modified: SystemTime) -> Outcome { + let modified = whole_seconds(modified); + match (&self.if_match, self.if_unmodified_since) { + (Some(list), _) if !names(list, etag) => return Outcome::Failed("If-Match"), + (None, Some(t)) if modified > whole_seconds(t) => { + return Outcome::Failed("If-Unmodified-Since"); + } + _ => {} + } + match (&self.if_none_match, self.if_modified_since) { + (Some(list), _) if names(list, etag) => Outcome::NotModified, + (None, Some(t)) if modified <= whole_seconds(t) => Outcome::NotModified, + _ => Outcome::Serve, + } + } +} + +fn whole_seconds(t: SystemTime) -> u64 { + t.duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or(Duration::ZERO) + .as_secs() +} + +/// A write's conditional headers. +#[derive(Debug, Default, PartialEq, Eq)] +pub(crate) struct WriteConditions { + /// `If-None-Match: *`: only where no object is + pub(crate) absent: bool, + /// `If-Match`: only over an object it names + pub(crate) if_match: Option, +} + +/// Why a write's conditional headers are not served: an `If-None-Match` other than `*`. +pub(crate) struct Unsupported; + +impl WriteConditions { + pub(crate) fn of(headers: &HeaderMap) -> Result { + let text = |name| { + let values: Vec<&str> = headers + .get_all(name) + .iter() + .map(|v| v.to_str().map_err(|_| Unsupported)) + .collect::>()?; + Ok((!values.is_empty()).then(|| values.join(","))) + }; + let absent = match text(IF_NONE_MATCH)? { + None => false, + Some(v) if v.trim() == "*" => true, + Some(_) => return Err(Unsupported), + }; + Ok(WriteConditions { + absent, + if_match: text(IF_MATCH)?, + }) + } + + /// What [`crate::files::commit`] checks, or `None` when nothing is asked. + pub(crate) fn precondition(self) -> Option { + if !self.absent && self.if_match.is_none() { + return None; + } + Some(Box::new(move |existing: Option<&Object>| { + (!self.absent || existing.is_none()) + && self + .if_match + .as_deref() + .is_none_or(|list| existing.is_some_and(|o| names(list, &etag_of(o)))) + })) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use hyper::header::HeaderValue; + + const SHA: &str = "0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0"; + const MD5: &str = "65a8e27d8879283831b664bd8b7f0ad4"; + + fn object(annotations: &[(&str, &str)]) -> Object { + Object { + repo: "files/b".into(), + tag: "k".into(), + layer: SHA.into(), + size: 13, + modified: at(1_000_000), + annotations: annotations + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + } + } + + fn at(secs: u64) -> SystemTime { + SystemTime::UNIX_EPOCH + Duration::from_secs(secs) + } + + fn headers(pairs: &[(hyper::header::HeaderName, &str)]) -> HeaderMap { + let mut h = HeaderMap::new(); + for (n, v) in pairs { + h.append(n.clone(), HeaderValue::from_str(v).unwrap()); + } + h + } + + #[test] + fn an_etag_is_the_recorded_md5_or_the_sha256() { + assert_eq!( + etag_of(&object(&[(MD5_ANNOTATION, MD5)])), + format!("\"{MD5}\"") + ); + assert_eq!(etag_of(&object(&[])), format!("\"{SHA}\"")); + assert_eq!( + etag_of(&object(&[(MD5_ANNOTATION, "\"x\"")])), + format!("\"{SHA}\"") + ); + let upper = MD5.to_ascii_uppercase(); + assert_eq!( + etag_of(&object(&[(MD5_ANNOTATION, &upper)])), + format!("\"{SHA}\"") + ); + assert_eq!( + etag_of(&object(&[(MD5_ANNOTATION, &MD5[1..])])), + format!("\"{SHA}\"") + ); + } + + #[test] + fn a_list_names_an_etag_quoted_weak_or_bare() { + let e = format!("\"{MD5}\""); + assert!(names("*", &e)); + assert!(names(&e, &e)); + assert!(names(&format!("W/{e}"), &e)); + assert!(names(MD5, &e)); + assert!(names(&format!("\"other\", {e}"), &e)); + assert!(!names("\"other\"", &e)); + assert!(!names("", &e)); + assert!(!names(&format!("\"{MD5}"), &e)); + } + + /// The eight combinations a read's headers can take on one object, and the order they + /// are weighed in. + #[test] + fn reads_are_decided_in_rfc_order() { + let e = format!("\"{MD5}\""); + let modified = at(1_000_000) + Duration::from_millis(700); + let before = httpdate::fmt_http_date(at(999_999)); + let same = httpdate::fmt_http_date(at(1_000_000)); + let decide = |pairs: &[(hyper::header::HeaderName, &str)]| { + Conditions::of(&headers(pairs)).evaluate(&e, modified) + }; + assert_eq!(decide(&[]), Outcome::Serve); + // If-Match + assert_eq!(decide(&[(IF_MATCH, &e)]), Outcome::Serve); + assert_eq!(decide(&[(IF_MATCH, "\"x\"")]), Outcome::Failed("If-Match")); + // If-Unmodified-Since, to the second + assert_eq!(decide(&[(IF_UNMODIFIED_SINCE, &same)]), Outcome::Serve); + assert_eq!( + decide(&[(IF_UNMODIFIED_SINCE, &before)]), + Outcome::Failed("If-Unmodified-Since") + ); + // a matching If-Match overrides a failing If-Unmodified-Since + assert_eq!( + decide(&[(IF_MATCH, &e), (IF_UNMODIFIED_SINCE, &before)]), + Outcome::Serve + ); + // If-None-Match + assert_eq!(decide(&[(IF_NONE_MATCH, &e)]), Outcome::NotModified); + assert_eq!(decide(&[(IF_NONE_MATCH, "*")]), Outcome::NotModified); + assert_eq!(decide(&[(IF_NONE_MATCH, "\"x\"")]), Outcome::Serve); + // If-Modified-Since, to the second + assert_eq!(decide(&[(IF_MODIFIED_SINCE, &same)]), Outcome::NotModified); + assert_eq!(decide(&[(IF_MODIFIED_SINCE, &before)]), Outcome::Serve); + // a non-matching If-None-Match overrides a met If-Modified-Since; a matching one + // answers 304 whatever the date says + assert_eq!( + decide(&[(IF_NONE_MATCH, "\"x\""), (IF_MODIFIED_SINCE, &same)]), + Outcome::Serve + ); + assert_eq!( + decide(&[(IF_NONE_MATCH, &e), (IF_MODIFIED_SINCE, &before)]), + Outcome::NotModified + ); + // a failed precondition outranks a 304 + assert_eq!( + decide(&[(IF_MATCH, "\"x\""), (IF_NONE_MATCH, &e)]), + Outcome::Failed("If-Match") + ); + // a date that does not parse is no condition + assert_eq!(decide(&[(IF_MODIFIED_SINCE, "yesterday")]), Outcome::Serve); + assert_eq!( + decide(&[(IF_UNMODIFIED_SINCE, "yesterday")]), + Outcome::Serve + ); + } + + #[test] + fn writes_create_only_or_replace_only_what_they_name() { + let md5 = object(&[(MD5_ANNOTATION, MD5)]); + let sha = object(&[]); + let check = |pairs: &[(hyper::header::HeaderName, &str)], existing: Option<&Object>| { + WriteConditions::of(&headers(pairs)) + .ok() + .unwrap() + .precondition() + .is_none_or(|p| p(existing)) + }; + assert!( + WriteConditions::of(&headers(&[])) + .ok() + .unwrap() + .precondition() + .is_none() + ); + assert!(check(&[], Some(&md5))); + // create only + assert!(check(&[(IF_NONE_MATCH, "*")], None)); + assert!(!check(&[(IF_NONE_MATCH, "*")], Some(&md5))); + // compare and swap, on either ETag form + let e = format!("\"{MD5}\""); + assert!(check(&[(IF_MATCH, &e)], Some(&md5))); + assert!(!check(&[(IF_MATCH, &e)], Some(&sha))); + assert!(check(&[(IF_MATCH, &format!("\"{SHA}\""))], Some(&sha))); + assert!(!check(&[(IF_MATCH, &e)], None)); + assert!(check(&[(IF_MATCH, "*")], Some(&sha))); + assert!(!check(&[(IF_MATCH, "*")], None)); + // only `*` is served for If-None-Match + assert!(WriteConditions::of(&headers(&[(IF_NONE_MATCH, &e)])).is_err()); + } +} diff --git a/vk-registry/src/s3/meta.rs b/vk-registry/src/s3/meta.rs new file mode 100644 index 00000000..b4fe2c5c --- /dev/null +++ b/vk-registry/src/s3/meta.rs @@ -0,0 +1,160 @@ +//! User metadata: the `x-amz-meta-` headers of a write, kept as manifest annotations +//! `org.virtkit.s3-meta.` and sent back on reads. Sync tools keep a file's own +//! modification time there. + +use std::collections::BTreeMap; + +use hyper::header::{HeaderMap, HeaderName, HeaderValue}; + +/// The prefix of a user metadata header. +const HEADER_PREFIX: &str = "x-amz-meta-"; + +/// The prefix of the annotation each one is kept as. +const ANNOTATION_PREFIX: &str = "org.virtkit.s3-meta."; + +/// Most bytes of user metadata, names and values together: S3's own bound. +pub(crate) const MAX_METADATA: usize = 2048; + +/// Why a write's metadata is refused: an S3 error code and the rule. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Refusal { + pub(crate) code: &'static str, + pub(crate) message: &'static str, +} + +/// The annotations a write's `x-amz-meta-*` headers become. A name is `[a-z0-9-]`, matched +/// without regard to case (header names arrive lowercase) and kept lowercase; a value is text; +/// a name sent twice is one value, the two joined by a comma, as HTTP joins them. +pub(crate) fn annotations(headers: &HeaderMap) -> Result, Refusal> { + let mut out = BTreeMap::new(); + let mut total = 0usize; + for name in headers.keys() { + let Some(short) = name.as_str().strip_prefix(HEADER_PREFIX) else { + continue; + }; + let short = short.to_ascii_lowercase(); + if short.is_empty() + || !short + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') + { + return Err(Refusal { + code: "InvalidArgument", + message: "a metadata name is x-amz-meta- then a run of a-z 0-9 -", + }); + } + let values = headers + .get_all(name) + .iter() + .map(|v| v.to_str()) + .collect::, _>>() + .map_err(|_| Refusal { + code: "InvalidArgument", + message: "a metadata value is printable ASCII", + })?; + let value = values.join(","); + total = total + .saturating_add(short.len()) + .saturating_add(value.len()); + if total > MAX_METADATA { + return Err(Refusal { + code: "MetadataTooLarge", + message: "user metadata is at most 2 KiB, names and values together", + }); + } + out.insert(format!("{ANNOTATION_PREFIX}{short}"), value); + } + Ok(out) +} + +/// The `x-amz-meta-*` headers an object's annotations hold; one that cannot be a header any +/// more is left out. +pub(crate) fn headers( + annotations: &BTreeMap, +) -> impl Iterator + '_ { + annotations.iter().filter_map(|(k, v)| { + let short = k.strip_prefix(ANNOTATION_PREFIX)?; + let name = HeaderName::from_bytes(format!("{HEADER_PREFIX}{short}").as_bytes()).ok()?; + Some((name, HeaderValue::from_str(v).ok()?)) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn of(pairs: &[(&str, &str)]) -> Result, Refusal> { + let mut h = HeaderMap::new(); + for (n, v) in pairs { + h.append( + HeaderName::from_bytes(n.as_bytes()).unwrap(), + HeaderValue::from_str(v).unwrap(), + ); + } + annotations(&h) + } + + #[test] + fn metadata_round_trips_through_annotations() { + let a = of(&[ + ("X-Amz-Meta-Mtime", "1700000000"), + ("x-amz-meta-a-1", "x"), + ("x-amz-meta-a-1", "y"), + ("content-type", "text/html"), + ("x-amz-date", "20260101T000000Z"), + ]) + .unwrap(); + assert_eq!( + a, + BTreeMap::from([ + ("org.virtkit.s3-meta.a-1".to_string(), "x,y".to_string()), + ( + "org.virtkit.s3-meta.mtime".to_string(), + "1700000000".to_string() + ), + ]) + ); + let mut all = a.clone(); + all.insert("org.virtkit.md5".into(), "00".into()); + let back: Vec<(String, String)> = headers(&all) + .map(|(n, v)| (n.to_string(), v.to_str().unwrap().to_string())) + .collect(); + assert_eq!( + back, + [ + ("x-amz-meta-a-1".to_string(), "x,y".to_string()), + ("x-amz-meta-mtime".to_string(), "1700000000".to_string()), + ] + ); + assert_eq!(of(&[]).unwrap(), BTreeMap::new()); + } + + #[test] + fn names_and_sizes_are_bounded() { + for bad in ["x-amz-meta-", "x-amz-meta-a_b", "x-amz-meta-a.b"] { + assert_eq!( + of(&[(bad, "v")]).unwrap_err().code, + "InvalidArgument", + "{bad}" + ); + } + let mut h = HeaderMap::new(); + h.insert("x-amz-meta-v", HeaderValue::from_bytes(b"caf\xe9").unwrap()); + assert_eq!(annotations(&h).unwrap_err().code, "InvalidArgument"); + // 2 KiB exactly, names counted without their prefix + let fill = "v".repeat(MAX_METADATA - 2); + assert!(of(&[("x-amz-meta-ab", &fill)]).is_ok()); + let over = "v".repeat(MAX_METADATA - 1); + assert_eq!( + of(&[("x-amz-meta-ab", &over)]).unwrap_err().code, + "MetadataTooLarge" + ); + let half = "v".repeat(MAX_METADATA / 2); + assert_eq!( + of(&[("x-amz-meta-a", &half), ("x-amz-meta-b", &half)]) + .unwrap_err() + .code, + "MetadataTooLarge" + ); + } +} diff --git a/vk-registry/src/s3/mod.rs b/vk-registry/src/s3/mod.rs index cf66de58..eab98bec 100644 --- a/vk-registry/src/s3/mod.rs +++ b/vk-registry/src/s3/mod.rs @@ -5,7 +5,17 @@ //! nothing new is stored; or by the bearer, Basic or session credential the other routes //! take. Every answer carries an `x-amz-request-id`, and refusals are S3's XML `Error`. //! Presigned URLs and temporary credentials are refused. +//! +//! Past the gate, a bucket is a top-level directory of `files/` and an object a key below +//! one, served over the object layer the WebDAV view shares ([`crate::files`]): [`bucket`] +//! answers the service and its buckets, [`object`] the objects. Operations not served yet +//! answer `NotImplemented`. +mod bucket; +mod conditional; +mod meta; +mod object; +mod path; mod sigv4; mod xml; @@ -17,6 +27,7 @@ use hyper::header::{ use hyper::{Method, Request, Response, StatusCode}; use vk_hub_proto::token::Claims; +use crate::files::{MAX_OBJECT, PutError}; use crate::{Authenticator, Authz, Body, ServerState, accounts, hubtoken}; /// Whether `path` is one this module answers. @@ -35,12 +46,11 @@ const REQUEST_ID: &str = "x-amz-request-id"; /// Serve one `/s3/…` request, from its credentials on: [`crate::route`] hands every such /// request here before its own gate when S3 is on. pub(crate) async fn route(state: &ServerState, req: Request) -> Result> { - let id = accounts::random_token(8); - let path = req.uri().path().to_string(); let ctx = Ctx { - id: &id, - path: &path, + id: accounts::random_token(8), + path: req.uri().path().to_string(), }; + let id = &ctx.id; let mut resp = match serve(state, req, &ctx).await { Ok(resp) => resp, Err(e) => { @@ -49,24 +59,49 @@ pub(crate) async fn route(state: &ServerState, req: Request) -> Result } }; let headers = resp.headers_mut(); - headers.insert(REQUEST_ID, HeaderValue::from_str(&id)?); + headers.insert(REQUEST_ID, HeaderValue::from_str(id)?); headers.insert(X_CONTENT_TYPE_OPTIONS, HeaderValue::from_static("nosniff")); Ok(resp) } /// What every answer to a request names: its id and the path it asked for. -struct Ctx<'a> { - id: &'a str, - path: &'a str, +#[derive(Clone)] +struct Ctx { + id: String, + path: String, } -impl Ctx<'_> { +impl Ctx { fn error(&self, status: StatusCode, code: &str, message: &str) -> Response { - xml::error(status, code, message, &[], self.path, self.id) + xml::error(status, code, message, &[], &self.path, &self.id) + } + + fn not_implemented(&self) -> Response { + self.error( + StatusCode::NOT_IMPLEMENTED, + "NotImplemented", + "this registry does not serve that operation", + ) + } + + fn access_denied(&self) -> Response { + self.error( + StatusCode::FORBIDDEN, + "AccessDenied", + "the credential grants nothing for this bucket or key", + ) + } + + fn too_large(&self) -> Response { + self.error( + StatusCode::PAYLOAD_TOO_LARGE, + "EntityTooLarge", + "an object is at most 4 GiB", + ) } } -fn internal_error(ctx: &Ctx<'_>) -> Response { +fn internal_error(ctx: &Ctx) -> Response { ctx.error( StatusCode::INTERNAL_SERVER_ERROR, "InternalError", @@ -92,11 +127,7 @@ impl Who { } } -async fn serve( - state: &ServerState, - req: Request, - ctx: &Ctx<'_>, -) -> Result> { +async fn serve(state: &ServerState, req: Request, ctx: &Ctx) -> Result> { let headers = req.headers(); // Browsers attach cookies or Basic credentials to cross-site requests, as on every // other machine route. @@ -139,7 +170,7 @@ async fn serve( }; // A hub managing the registry may make it refuse writes for a while. - if state.store.read_only() && crate::is_store_write(req.method(), ctx.path) { + if state.store.read_only() && crate::is_store_write(req.method(), &ctx.path) { let mut resp = ctx.error( StatusCode::SERVICE_UNAVAILABLE, "ServiceUnavailable", @@ -147,40 +178,88 @@ async fn serve( ); resp.headers_mut() .insert(RETRY_AFTER, HeaderValue::from_static("60")); - return crate::dav::refuse(req, resp, ctx.path).await; + return refuse(req, resp, ctx).await; } - unserved(&who.authz(), req, ctx).await + operate(state, &who.authz(), req, ctx).await +} + +/// The query parameter an AWS SDK adds to name the operation it calls, which says nothing +/// the method and path do not. +const OPERATION_HINT: &str = "x-id"; + +/// The subresource a query names (`location`, `attributes`, `uploads`…): `None` for a query +/// of [`OPERATION_HINT`] at most, `Some("")` when it names more than one parameter or names +/// one with a value — which nothing served here takes. +fn subresource(query: &str) -> Option<&str> { + let mut names = query + .split('&') + .filter(|p| !p.is_empty()) + .filter(|p| p.split_once('=').map_or(*p, |(n, _)| n) != OPERATION_HINT); + let first = names.next()?; + Some(match (first.split_once('='), names.next()) { + (None | Some((_, "")), None) => first.trim_end_matches('='), + _ => "", + }) } -/// The answer to an authenticated request while no S3 operation is served: a read of an -/// object or a bucket finds nothing, whoever `_authz` names, and anything else is not -/// implemented. The body is read through first, as after any authenticated refusal. -async fn unserved( - _authz: &Authz<'_>, +/// An authenticated request, to the operation its method, path and query name. What is not +/// served answers `NotImplemented` once the body is read, as every refusal past the gate does. +async fn operate( + state: &ServerState, + authz: &Authz<'_>, req: Request, - ctx: &Ctx<'_>, + ctx: &Ctx, ) -> Result> { - let rest = ctx.path.strip_prefix("/s3").unwrap_or_default(); - let rest = rest.strip_prefix('/').unwrap_or(rest); - let read = matches!(req.method(), &Method::GET | &Method::HEAD); - let resp = match rest.split_once('/') { - _ if !read || rest.is_empty() => ctx.error( - StatusCode::NOT_IMPLEMENTED, - "NotImplemented", - "this registry does not serve that operation", - ), - None | Some((_, "")) => ctx.error( - StatusCode::NOT_FOUND, - "NoSuchBucket", - "the specified bucket does not exist", - ), - Some(_) => ctx.error( - StatusCode::NOT_FOUND, - "NoSuchKey", - "the specified key does not exist", - ), + let target = match path::parse(&ctx.path) { + Ok(target) => target, + Err(why) => { + let resp = ctx.error(StatusCode::BAD_REQUEST, why.code, why.message); + return refuse(req, resp, ctx).await; + } }; - crate::dav::refuse(req, resp, ctx.path).await + let query = req.uri().query().unwrap_or("").to_string(); + let sub = subresource(&query); + let method = req.method().clone(); + match (target, &method, sub) { + (path::Target::Service, &Method::GET, None) => bucket::list(state, authz, req, ctx).await, + (path::Target::Bucket(b), &Method::HEAD, None) => { + bucket::head(state, authz, b, req, ctx).await + } + (path::Target::Bucket(b), &Method::GET, Some("location")) => { + bucket::location(state, authz, b, req, ctx).await + } + (path::Target::Object { segs, .. }, &Method::GET | &Method::HEAD, None) => { + object::get(state, authz, segs, method == Method::HEAD, req, ctx).await + } + (path::Target::Object { segs, .. }, &Method::GET, Some("attributes")) => { + object::attributes(state, authz, segs, req, ctx).await + } + (path::Target::Object { segs, .. }, &Method::PUT, None) => { + object::put(state, authz, segs, req, ctx).await + } + (path::Target::Object { segs, .. }, &Method::DELETE, None) => { + object::delete(state, authz, segs, req, ctx).await + } + _ => refuse(req, ctx.not_implemented(), ctx).await, + } +} + +/// Answer `resp` once the request body has been read through, as after every refusal of an +/// authenticated request: a status sent with request bytes unread resets the connection, +/// which a client sees instead of the answer. A body past the object cap is refused with 413 +/// instead. +async fn refuse(req: Request, resp: Response, ctx: &Ctx) -> Result> { + Ok(drain(req, ctx).await?.unwrap_or(resp)) +} + +/// Read a request's body through and discard it: `Some(413)` if it crosses the object cap. +async fn drain(req: Request, ctx: &Ctx) -> Result>> { + let sink = &mut tokio::io::sink(); + match crate::files::drain_into(sink, req.into_body(), MAX_OBJECT, &mut |_| {}).await { + Ok(_) => Ok(None), + Err(PutError::TooLarge) => Ok(Some(ctx.too_large())), + Err(PutError::Failed(e)) => Err(e), + } } /// Answer `resp` to a request refused before it was authenticated, having read at most @@ -207,7 +286,7 @@ fn is_presigned(query: &str) -> bool { fn presented( state: &ServerState, req: &Request, - ctx: &Ctx<'_>, + ctx: &Ctx, ) -> std::result::Result>> { let denied = || { let message = if req.headers().contains_key(AUTHORIZATION) { @@ -255,7 +334,7 @@ fn token( /// The answer when the accounts database could not be read: the release taking over is /// asked again, anything else is logged under the request's id. -fn resolve_failed(e: anyhow::Error, ctx: &Ctx<'_>) -> Box> { +fn resolve_failed(e: anyhow::Error, ctx: &Ctx) -> Box> { if accounts::is_handed_over_error(&e) { return Box::new(crate::ask_again(ctx.error( StatusCode::SERVICE_UNAVAILABLE, @@ -279,7 +358,7 @@ fn resolve_failed(e: anyhow::Error, ctx: &Ctx<'_>) -> Box> { fn signed_by( state: &ServerState, req: &Request, - ctx: &Ctx<'_>, + ctx: &Ctx, ) -> std::result::Result>> { let headers = req.headers(); let malformed = |why: &str| { @@ -326,8 +405,8 @@ fn signed_by( ("ServerTime", &server_time), ("MaxAllowedSkewMilliseconds", &skew), ], - ctx.path, - ctx.id, + &ctx.path, + &ctx.id, ))); } } @@ -390,8 +469,8 @@ fn signed_by( "SignatureDoesNotMatch", "the request signature does not match the one computed with the key's secret", &[("StringToSign", &sts), ("CanonicalRequest", &canonical)], - ctx.path, - ctx.id, + &ctx.path, + &ctx.id, ))); } Ok(who) @@ -433,6 +512,23 @@ mod tests { assert!(!is_presigned("")); } + #[test] + fn a_query_names_one_subresource_or_none() { + assert_eq!(subresource(""), None); + assert_eq!(subresource("x-id=GetObject"), None); + assert_eq!(subresource("location"), Some("location")); + assert_eq!(subresource("location="), Some("location")); + assert_eq!( + subresource("attributes&x-id=GetObjectAttributes"), + Some("attributes") + ); + assert_eq!(subresource("x-id=PutObject&tagging"), Some("tagging")); + assert_eq!(subresource("list-type=2"), Some("")); + assert_eq!(subresource("partNumber=1&uploadId=a"), Some("")); + assert_eq!(subresource("location&acl"), Some("")); + assert_eq!(subresource("versionId=null"), Some("")); + } + #[test] fn the_id_becomes_one_bearer_header_or_nothing() { let h = bearer_of("vkr_abc").unwrap(); diff --git a/vk-registry/src/s3/object.rs b/vk-registry/src/s3/object.rs new file mode 100644 index 00000000..42d2337a --- /dev/null +++ b/vk-registry/src/s3/object.rs @@ -0,0 +1,654 @@ +//! Objects: PutObject, GetObject and HeadObject, GetObjectAttributes and DeleteObject, over +//! the object layer the WebDAV view shares. Authorized as that view authorizes: a write needs +//! `Write` on the key's parent repository, a read `Read` on the repository the key resolves +//! to; a caller holding the action on neither the key's path nor its parent is refused before +//! any lookup, and what it may not read is absent (`NoSuchKey`). +//! +//! An object is served as bytes and nothing else: `application/octet-stream`, `attachment` +//! and `nosniff` whatever its writer said, since this origin also serves `/browse` and holds +//! the session cookie. Its ETag is the MD5 its S3 write recorded, or its SHA-256 (see +//! [`conditional::etag`]). + +use std::sync::Arc; + +use anyhow::{Context, Result}; +use hyper::body::Incoming; +use hyper::header::{ + CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LENGTH, ETAG, HeaderMap, HeaderValue, + LAST_MODIFIED, +}; +use hyper::{Request, Response, StatusCode}; +use sha1::Digest as _; + +use super::conditional::{self, Conditions, MD5_ANNOTATION, Outcome, WriteConditions}; +use super::{Ctx, drain, meta, refuse, xml}; +use crate::dav::http_date; +use crate::files::{ + self, Commit, MAX_OBJECT, Node, Object, PutError, Unwritable, Written, allowed, object_of, + repo_name, +}; +use crate::{Authz, Body, ServerState, Store, accounts, body_of}; + +/// The header asking a read for the object's checksum. +const CHECKSUM_MODE: &str = "x-amz-checksum-mode"; + +/// The checksum a read answers with, when asked: the SHA-256 the store names the bytes by. +const CHECKSUM_SHA256: &str = "x-amz-checksum-sha256"; + +/// Whether this caller may `action` on the key at `segs` or on its parent: what admits a +/// request to the lookup that then decides on the repository the key resolves to. +fn admitted(authz: &Authz<'_>, action: accounts::Action, segs: &[String]) -> bool { + allowed(authz, action, &repo_name(segs)) + || object_of(segs).is_some_and(|(parent, _)| allowed(authz, action, &parent)) +} + +fn no_such_key(ctx: &Ctx) -> Response { + ctx.error( + StatusCode::NOT_FOUND, + "NoSuchKey", + "the specified key does not exist", + ) +} + +/// The object at `segs` as this caller may read it: `None` when nothing is there, when a +/// directory or a foreign tag is, or when the caller may not read the repository it is in or +/// its bytes through it. +fn readable(store: &Store, authz: &Authz<'_>, segs: &[String]) -> Result> { + Ok(match files::resolve(store, segs)? { + Some(Node::Object(obj)) + if allowed(authz, accounts::Action::Read, &obj.repo) + && crate::readable_through(authz, store, &obj.repo, &obj.layer) => + { + Some(obj) + } + _ => None, + }) +} + +/// The SHA-256 of an object, as S3's checksum headers and elements carry it: base64. +fn sha256_base64(layer: &str) -> String { + vk_hub_proto::to_base64(&hex_bytes(layer).unwrap_or_default()) +} + +fn hex_bytes(hex: &str) -> Option> { + let b = hex.as_bytes(); + if !b.len().is_multiple_of(2) { + return None; + } + b.chunks(2) + .map(|pair| { + std::str::from_utf8(pair) + .ok() + .and_then(|p| u8::from_str_radix(p, 16).ok()) + }) + .collect() +} + +fn insert(headers: &mut HeaderMap, name: hyper::header::HeaderName, value: &str) { + if let Ok(v) = HeaderValue::from_str(value) { + headers.insert(name, v); + } +} + +/// `GET`/`HEAD /s3//`: the object's bytes, after its conditional headers. A GET +/// served refreshes an idle object's tag, as a WebDAV GET does; a HEAD touches nothing. +/// `Range` is ignored: a whole 200 is a legal answer to it. +pub(super) async fn get( + state: &ServerState, + authz: &Authz<'_>, + segs: Vec, + head: bool, + req: Request, + ctx: &Ctx, +) -> Result> { + if !admitted(authz, accounts::Action::Read, &segs) { + return refuse(req, ctx.access_denied(), ctx).await; + } + let conditions = Conditions::of(req.headers()); + let checksum = req + .headers() + .get(CHECKSUM_MODE) + .is_some_and(|v| v.as_bytes().eq_ignore_ascii_case(b"ENABLED")); + let ctx = ctx.clone(); + crate::dav::blocking(state, authz, move |store, authz| { + let Some(obj) = readable(store, authz, &segs)? else { + return Ok(no_such_key(&ctx)); + }; + let etag = conditional::etag_of(&obj); + let modified = http_date(obj.modified); + match conditions.evaluate(&etag, obj.modified) { + Outcome::Serve => {} + Outcome::NotModified => { + let mut resp = Response::builder() + .status(StatusCode::NOT_MODIFIED) + .body(body_of(bytes::Bytes::new()))?; + insert(resp.headers_mut(), ETAG, &etag); + insert(resp.headers_mut(), LAST_MODIFIED, &modified); + return Ok(resp); + } + Outcome::Failed(condition) => { + return Ok(xml::error( + StatusCode::PRECONDITION_FAILED, + "PreconditionFailed", + "at least one of the preconditions you specified did not hold", + &[("Condition", condition)], + &ctx.path, + &ctx.id, + )); + } + } + let mut resp = + crate::serve_blob(store, &format!("sha256:{}", obj.layer), head, false, false)?; + // The layer gone under the tag (a gc between the two reads) is no object either. + if resp.status() != StatusCode::OK { + return Ok(no_such_key(&ctx)); + } + if !head { + files::touch_if_idle(store, &obj); + } + let headers = resp.headers_mut(); + headers.remove("docker-content-digest"); + headers.insert(CONTENT_DISPOSITION, HeaderValue::from_static("attachment")); + insert(headers, ETAG, &etag); + insert(headers, LAST_MODIFIED, &modified); + if checksum { + insert( + headers, + hyper::header::HeaderName::from_static(CHECKSUM_SHA256), + &sha256_base64(&obj.layer), + ); + } + for (name, value) in meta::headers(&obj.annotations) { + headers.append(name, value); + } + Ok(resp) + }) + .await +} + +/// `GET /s3//?attributes`: what `x-amz-object-attributes` asks of `ETag`, +/// `Checksum`, `ObjectSize` and `StorageClass` (`ObjectParts` has nothing to say of an object +/// written whole), with `Last-Modified`. A metadata read, so it touches nothing. +pub(super) async fn attributes( + state: &ServerState, + authz: &Authz<'_>, + segs: Vec, + req: Request, + ctx: &Ctx, +) -> Result> { + if !admitted(authz, accounts::Action::Read, &segs) { + return refuse(req, ctx.access_denied(), ctx).await; + } + let mut asked = Vec::new(); + for value in req.headers().get_all("x-amz-object-attributes") { + let Ok(value) = value.to_str() else { + asked.clear(); + break; + }; + asked.extend(value.split(',').map(|a| a.trim().to_string())); + } + let known = [ + "ETag", + "Checksum", + "ObjectParts", + "StorageClass", + "ObjectSize", + ]; + if asked.is_empty() || asked.iter().any(|a| !known.contains(&a.as_str())) { + let resp = ctx.error( + StatusCode::BAD_REQUEST, + "InvalidArgument", + "x-amz-object-attributes names some of ETag, Checksum, ObjectParts, StorageClass \ + and ObjectSize", + ); + return refuse(req, resp, ctx).await; + } + let ctx = ctx.clone(); + crate::dav::blocking(state, authz, move |store, authz| { + let Some(obj) = readable(store, authz, &segs)? else { + return Ok(no_such_key(&ctx)); + }; + let has = |a: &str| asked.iter().any(|x| x == a); + let mut inner = String::new(); + if has("ETag") { + // Unquoted here, as S3 writes it in this document. + inner.push_str(&xml::element( + "ETag", + conditional::etag_of(&obj).trim_matches('"'), + )); + } + if has("Checksum") { + inner.push_str(&format!( + "{}", + xml::element("ChecksumSHA256", &sha256_base64(&obj.layer)) + )); + } + if has("StorageClass") { + inner.push_str(&xml::element("StorageClass", "STANDARD")); + } + if has("ObjectSize") { + inner.push_str(&xml::element("ObjectSize", &obj.size.to_string())); + } + let mut resp = xml::document("GetObjectAttributesResponse", &inner); + insert(resp.headers_mut(), LAST_MODIFIED, &http_date(obj.modified)); + Ok(resp) + }) + .await +} + +/// How a PutObject's body is bound to its signature, by `x-amz-content-sha256`. +enum Payload { + /// `UNSIGNED-PAYLOAD`, or no header (an unsigned request) + Unsigned, + /// the body's SHA-256, lowercase hex + Sha256(String), +} + +/// The digests a PutObject asks to be checked against what arrives, each as bytes. +#[derive(Default)] +struct Expected { + md5: Option>, + sha256: Option>, + sha1: Option>, + crc32: Option>, + /// the `x-amz-checksum-*` headers to echo once they held + echo: Vec<(&'static str, String)>, +} + +/// What a PutObject's headers ask that cannot be served, or are malformed: a status, a code +/// and a message. +type Refused = (StatusCode, &'static str, &'static str); + +fn payload(headers: &HeaderMap) -> std::result::Result { + let chunked = headers.get_all(CONTENT_ENCODING).iter().any(|v| { + v.to_str().is_ok_and(|v| { + v.split(',') + .any(|e| e.trim().eq_ignore_ascii_case("aws-chunked")) + }) + }); + let streaming = ( + StatusCode::NOT_IMPLEMENTED, + "NotImplemented", + "aws-chunked uploads are not served yet: send UNSIGNED-PAYLOAD or the body's SHA-256", + ); + if chunked { + return Err(streaming); + } + let Some(value) = headers.get("x-amz-content-sha256") else { + return Ok(Payload::Unsigned); + }; + let value = value.to_str().unwrap_or_default(); + if value == "UNSIGNED-PAYLOAD" { + return Ok(Payload::Unsigned); + } + if value.starts_with("STREAMING-") { + return Err(streaming); + } + if value.len() == 64 && value.bytes().all(|b| b.is_ascii_hexdigit()) { + return Ok(Payload::Sha256(value.to_ascii_lowercase())); + } + Err(( + StatusCode::BAD_REQUEST, + "InvalidArgument", + "x-amz-content-sha256 is UNSIGNED-PAYLOAD or the body's SHA-256 in hex", + )) +} + +fn expected(headers: &HeaderMap) -> std::result::Result { + // The header's text and the bytes it decodes to, `len` of them. + let decoded = |name: &str, len: usize| -> std::result::Result)>, ()> { + let Some(v) = headers.get(name) else { + return Ok(None); + }; + let text = v.to_str().map_err(|_| ())?.trim(); + let bytes = crate::auth::base64_decode(text) + .filter(|b| b.len() == len) + .ok_or(())?; + Ok(Some((text.to_string(), bytes))) + }; + let md5 = decoded("content-md5", 16).map_err(|()| { + ( + StatusCode::BAD_REQUEST, + "InvalidDigest", + "Content-MD5 is the body's MD5 in base64", + ) + })?; + let mut out = Expected { + md5: md5.map(|(_, b)| b), + ..Expected::default() + }; + for (name, len) in [ + ("x-amz-checksum-sha256", 32), + ("x-amz-checksum-sha1", 20), + ("x-amz-checksum-crc32", 4), + ] { + let value = decoded(name, len).map_err(|()| { + ( + StatusCode::BAD_REQUEST, + "InvalidRequest", + "an x-amz-checksum header is the checksum of the body in base64", + ) + })?; + if let Some((text, value)) = value { + out.echo.push((name, text)); + match name { + "x-amz-checksum-sha256" => out.sha256 = Some(value), + "x-amz-checksum-sha1" => out.sha1 = Some(value), + _ => out.crc32 = Some(value), + } + } + } + Ok(out) +} + +/// The digests computed while a body streams: MD5 always (the ETag), SHA-1 and CRC32 only +/// when the request asks to be checked against them. +struct Digests { + md5: md5::Context, + sha1: Option, + crc32: Option, +} + +impl Digests { + fn for_(expected: &Expected) -> Self { + Digests { + md5: md5::Context::new(), + sha1: expected.sha1.is_some().then(sha1::Sha1::new), + crc32: expected.crc32.is_some().then(crc32fast::Hasher::new), + } + } + + fn update(&mut self, data: &[u8]) { + self.md5.consume(data); + if let Some(h) = &mut self.sha1 { + h.update(data); + } + if let Some(h) = &mut self.crc32 { + h.update(data); + } + } +} + +/// The answer to a write at a path that takes no object, with what is in the way. +fn unwritable(why: Unwritable, ctx: &Ctx) -> Response { + let message = match why { + Unwritable::TopLevel => "a bucket takes no object of its own name", + Unwritable::Directory => "this key is a prefix of other keys, and cannot be an object", + Unwritable::Foreign => "this key holds content that is not a file, pushed over /v2/", + Unwritable::UnderObject => "a prefix of this key is an object, which cannot hold keys", + }; + ctx.error(StatusCode::CONFLICT, "OperationAborted", message) +} + +/// `PUT /s3//`: stream the body into staging while hashing it, check it against +/// what the request said of it, then tag it under the namespace lock, with its MD5 and user +/// metadata on the manifest and the request's `If-None-Match: *`/`If-Match` checked there. +/// Its `Content-Type` is not kept: every object is served as bytes. +pub(super) async fn put( + state: &ServerState, + authz: &Authz<'_>, + segs: Vec, + req: Request, + ctx: &Ctx, +) -> Result> { + let parent = object_of(&segs).map(|(parent, _)| parent); + if !parent.is_some_and(|p| allowed(authz, accounts::Action::Write, &p)) { + return refuse(req, ctx.access_denied(), ctx).await; + } + let headers = req.headers(); + // A declared length over the cap is refused unread; the connection then closes. + if headers + .get(CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()) + .is_some_and(|n| n > MAX_OBJECT) + { + return Ok(ctx.too_large()); + } + if headers.contains_key("x-amz-copy-source") { + return refuse(req, ctx.not_implemented(), ctx).await; + } + let checked = payload(headers).and_then(|p| Ok((p, expected(headers)?))); + let (payload, expected) = match checked { + Ok(checked) => checked, + Err((status, code, message)) => { + return refuse(req, ctx.error(status, code, message), ctx).await; + } + }; + let mut annotations = match meta::annotations(headers) { + Ok(a) => a, + Err(why) => { + let resp = ctx.error(StatusCode::BAD_REQUEST, why.code, why.message); + return refuse(req, resp, ctx).await; + } + }; + let Ok(conditions) = WriteConditions::of(headers) else { + let resp = ctx.error( + StatusCode::NOT_IMPLEMENTED, + "NotImplemented", + "a write's If-None-Match is * or nothing", + ); + return refuse(req, resp, ctx).await; + }; + // Refused before the body is staged when the answer is already known; checked again + // once it is, under the namespace lock. + let early = { + let (store, segs) = (Arc::clone(&state.store), segs.clone()); + tokio::task::spawn_blocking(move || files::unwritable(&store, &segs)) + .await + .context("checking a PUT panicked")?? + }; + if let Some(why) = early { + return refuse(req, unwritable(why, ctx), ctx).await; + } + + let mut digests = Digests::for_(&expected); + let staged = match files::stage(&state.store, req.into_body(), &mut |d| digests.update(d)).await + { + Ok(staged) => staged, + Err(PutError::TooLarge) => return Ok(ctx.too_large()), + Err(PutError::Failed(e)) => return Err(e), + }; + // A refusal from here on drops `staged`, which unlinks it: nothing is tagged. + if let Payload::Sha256(signed) = &payload + && *signed != staged.hex + { + return Ok(ctx.error( + StatusCode::BAD_REQUEST, + "XAmzContentSHA256Mismatch", + "the body's SHA-256 is not the x-amz-content-sha256 the request was signed with", + )); + } + let md5 = digests.md5.finalize().0; + let sha1 = digests.sha1.map(|h| h.finalize().to_vec()); + let crc32 = digests.crc32.map(|h| h.finalize().to_be_bytes().to_vec()); + if expected.md5.as_deref().is_some_and(|m| m != md5) { + return Ok(ctx.error( + StatusCode::BAD_REQUEST, + "BadDigest", + "the Content-MD5 you specified did not match what was received", + )); + } + let sha256 = hex_bytes(&staged.hex); + let mismatch = |expected: &Option>, got: &Option>| { + expected.as_ref().is_some_and(|e| Some(e) != got.as_ref()) + }; + if mismatch(&expected.sha256, &sha256) + || mismatch(&expected.sha1, &sha1) + || mismatch(&expected.crc32, &crc32) + { + return Ok(ctx.error( + StatusCode::BAD_REQUEST, + "BadDigest", + "the x-amz-checksum you specified did not match the checksum of what was received", + )); + } + let md5_hex = crate::hex_of(&md5); + annotations.insert(MD5_ANNOTATION.to_string(), md5_hex.clone()); + let how = Commit { + annotations, + precondition: conditions.precondition(), + }; + let written = files::commit(Arc::clone(&state.store), segs, staged, how).await?; + match written { + Written::Created | Written::Replaced => {} + Written::Refused(why) => return Ok(unwritable(why, ctx)), + Written::PreconditionFailed => { + return Ok(ctx.error( + StatusCode::PRECONDITION_FAILED, + "PreconditionFailed", + "at least one of the preconditions you specified did not hold", + )); + } + } + let mut resp = Response::builder() + .status(StatusCode::OK) + .header(CONTENT_LENGTH, "0") + .body(body_of(bytes::Bytes::new()))?; + let headers = resp.headers_mut(); + insert(headers, ETAG, &format!("\"{md5_hex}\"")); + for (name, value) in expected.echo { + insert( + headers, + hyper::header::HeaderName::from_static(name), + &value, + ); + } + Ok(resp) +} + +/// `DELETE /s3//`: 204 whether or not the key held an object, as S3 answers; +/// a key that is a directory has no object to delete, and is 204 too. A foreign tag is not +/// this view's to delete (409), and a caller that may read the object but not write it is +/// refused (403). Under the namespace lock, as every change of `files/` is. +pub(super) async fn delete( + state: &ServerState, + authz: &Authz<'_>, + segs: Vec, + req: Request, + ctx: &Ctx, +) -> Result> { + if !admitted(authz, accounts::Action::Write, &segs) { + return refuse(req, ctx.access_denied(), ctx).await; + } + if let Some(resp) = drain(req, ctx).await? { + return Ok(resp); + } + let ctx = ctx.clone(); + crate::dav::blocking(state, authz, move |store, authz| { + let _namespace = files::namespace(store)?; + let node = match files::resolve(store, &segs)? { + Some(node) if allowed(authz, accounts::Action::Read, node.repo()) => node, + // Nothing there, as far as this caller can tell. + _ => return Ok(no_content()), + }; + if !allowed(authz, accounts::Action::Write, node.repo()) { + return Ok(ctx.access_denied()); + } + match node { + Node::Object(obj) => { + store.delete_tag(&obj.repo, &obj.tag)?; + } + Node::Foreign(_) => { + return Ok(unwritable(Unwritable::Foreign, &ctx)); + } + Node::Dir(_) => {} + } + Ok(no_content()) + }) + .await +} + +fn no_content() -> Response { + Response::builder() + .status(StatusCode::NO_CONTENT) + .body(body_of(bytes::Bytes::new())) + .expect("building a 204") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn headers(pairs: &[(&str, &str)]) -> HeaderMap { + let mut h = HeaderMap::new(); + for (n, v) in pairs { + h.append( + hyper::header::HeaderName::from_bytes(n.as_bytes()).unwrap(), + HeaderValue::from_str(v).unwrap(), + ); + } + h + } + + #[test] + fn a_payload_is_unsigned_or_its_sha256_and_streaming_waits() { + let code = |pairs: &[(&str, &str)]| payload(&headers(pairs)).err().map(|e| e.1); + assert!(matches!(payload(&headers(&[])), Ok(Payload::Unsigned))); + assert!(matches!( + payload(&headers(&[("x-amz-content-sha256", "UNSIGNED-PAYLOAD")])), + Ok(Payload::Unsigned) + )); + let hex = "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855"; + assert!(matches!( + payload(&headers(&[("x-amz-content-sha256", hex)])), + Ok(Payload::Sha256(h)) if h == hex.to_ascii_lowercase() + )); + for streaming in [ + "STREAMING-AWS4-HMAC-SHA256-PAYLOAD", + "STREAMING-UNSIGNED-PAYLOAD-TRAILER", + ] { + assert_eq!( + code(&[("x-amz-content-sha256", streaming)]), + Some("NotImplemented") + ); + } + assert_eq!( + code(&[("content-encoding", "gzip, aws-chunked")]), + Some("NotImplemented") + ); + assert_eq!( + code(&[("x-amz-content-sha256", "abc")]), + Some("InvalidArgument") + ); + } + + #[test] + fn checksums_are_base64_of_their_length() { + let md5 = vk_hub_proto::to_base64(&[0u8; 16]); + let sha = vk_hub_proto::to_base64(&[1u8; 32]); + let e = expected(&headers(&[ + ("content-md5", &md5), + ("x-amz-checksum-sha256", &sha), + ("x-amz-checksum-crc32c", "AAAAAA=="), + ])) + .ok() + .unwrap(); + assert_eq!(e.md5, Some(vec![0u8; 16])); + assert_eq!(e.sha256, Some(vec![1u8; 32])); + assert_eq!(e.echo, [("x-amz-checksum-sha256", sha.clone())]); + let code = |pairs: &[(&str, &str)]| expected(&headers(pairs)).err().map(|e| e.1); + assert_eq!(code(&[("content-md5", "AAAA")]), Some("InvalidDigest")); + assert_eq!(code(&[("content-md5", "!!")]), Some("InvalidDigest")); + assert_eq!( + code(&[("x-amz-checksum-sha256", &md5)]), + Some("InvalidRequest") + ); + assert_eq!( + code(&[("x-amz-checksum-crc32", &md5)]), + Some("InvalidRequest") + ); + } + + #[test] + fn hex_decodes_whole_bytes_only() { + assert_eq!(hex_bytes("00ff10"), Some(vec![0, 255, 16])); + assert_eq!(hex_bytes("0"), None); + assert_eq!(hex_bytes("zz"), None); + let sha = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + assert_eq!( + sha256_base64(sha), + "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + ); + } +} diff --git a/vk-registry/src/s3/path.rs b/vk-registry/src/s3/path.rs new file mode 100644 index 00000000..c8a26690 --- /dev/null +++ b/vk-registry/src/s3/path.rs @@ -0,0 +1,259 @@ +//! The path of an `/s3/` request as what it names in `files/`: the service, a bucket (a +//! top-level directory) or an object (a key below one). Split before percent-decoding, so an +//! escaped `/` cannot make one component two, and decoded as the WebDAV view decodes its +//! paths. A key maps onto the store's names as they are, so it has to be one: each component +//! a repository name's component, the leaf a tag, no empty component and no trailing slash. +//! Arbitrary keys would need an escaping scheme, which this view deliberately does not have. + +use crate::dav::decode_segment; + +/// Most components a key may have: a repository name's bound less `files` and the bucket. +pub(crate) const MAX_KEY_COMPONENTS: usize = crate::MAX_NAME_SEGMENTS - 2; + +/// Longest key, in bytes once decoded: S3's own bound. +pub(crate) const MAX_KEY: usize = 1024; + +/// Longest key component, in bytes: a filename's. +const MAX_COMPONENT: usize = 255; + +/// What a path names. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Target { + /// `/s3` or `/s3/` + Service, + /// `/s3/`, with or without a trailing slash + Bucket(String), + /// `/s3//`: `segs` are the bucket then the key's components, the path below + /// `files/` the object layer takes + Object { key: String, segs: Vec }, +} + +/// Why a path names nothing this view serves: an S3 error code and the rule it broke. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Refusal { + pub(crate) code: &'static str, + pub(crate) message: &'static str, +} + +const fn refusal(code: &'static str, message: &'static str) -> Refusal { + Refusal { code, message } +} + +const BAD_BUCKET: Refusal = refusal( + "InvalidBucketName", + "a bucket name is one run of A-Z a-z 0-9 . _ - and not tags, manifests or blobs", +); +const BAD_ESCAPE: Refusal = refusal( + "InvalidArgument", + "the key has a malformed percent escape or is not UTF-8", +); +const EMPTY_COMPONENT: Refusal = refusal( + "InvalidArgument", + "a key has no empty component and does not end in a slash", +); +const BAD_CHARACTER: Refusal = refusal( + "InvalidArgument", + "a key's components are runs of A-Z a-z 0-9 . _ - only", +); +const DOT_COMPONENT: Refusal = refusal("InvalidArgument", "a key component cannot be . or .."); +const RESERVED_COMPONENT: Refusal = refusal( + "InvalidArgument", + "a key component cannot be tags, manifests or blobs", +); +const TOO_DEEP: Refusal = refusal("InvalidArgument", "a key has at most 14 components"); +const COMPONENT_TOO_LONG: Refusal = + refusal("KeyTooLongError", "a key component is at most 255 bytes"); +const KEY_TOO_LONG: Refusal = refusal("KeyTooLongError", "a key is at most 1024 bytes"); + +/// What the path of a request under `/s3` names; the caller dispatches only `/s3` paths here. +pub(crate) fn parse(path: &str) -> Result { + let rest = path.strip_prefix("/s3").unwrap_or(path); + let rest = rest.strip_prefix('/').unwrap_or(rest); + if rest.is_empty() { + return Ok(Target::Service); + } + let (raw_bucket, raw_key) = match rest.split_once('/') { + Some((b, k)) => (b, Some(k)), + None => (rest, None), + }; + let bucket = decode_segment(raw_bucket) + .filter(|b| crate::valid_name(b) && !b.contains('/') && b.len() <= MAX_COMPONENT) + .ok_or(BAD_BUCKET)?; + let raw_key = match raw_key { + None | Some("") => return Ok(Target::Bucket(bucket)), + Some(k) => k, + }; + let mut segs = vec![bucket]; + for raw in raw_key.split('/') { + if segs.len() > MAX_KEY_COMPONENTS { + return Err(TOO_DEEP); + } + segs.push(component(raw)?); + } + let key = segs.get(1..).unwrap_or_default().join("/"); + if key.len() > MAX_KEY { + return Err(KEY_TOO_LONG); + } + Ok(Target::Object { key, segs }) +} + +/// One key component, decoded and held to a repository name's component rule (which the +/// leaf, as a tag, also passes). +fn component(raw: &str) -> Result { + if raw.is_empty() { + return Err(EMPTY_COMPONENT); + } + let seg = decode_segment(raw).ok_or(BAD_ESCAPE)?; + if seg.len() > MAX_COMPONENT { + return Err(COMPONENT_TOO_LONG); + } + if seg.is_empty() { + return Err(EMPTY_COMPONENT); + } + if !seg + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) + { + return Err(BAD_CHARACTER); + } + if seg == "." || seg == ".." { + return Err(DOT_COMPONENT); + } + if !crate::valid_name(&seg) { + return Err(RESERVED_COMPONENT); + } + Ok(seg) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn object(key: &str) -> Result { + parse(&format!("/s3/b/{key}")) + } + + #[track_caller] + fn refused(key: &str, code: &str) -> &'static str { + match object(key) { + Err(r) => { + assert_eq!(r.code, code, "{key:?}: {}", r.message); + r.message + } + Ok(t) => panic!("accepted {key:?} as {t:?}"), + } + } + + #[test] + fn the_service_and_buckets_are_named_by_one_component() { + assert_eq!(parse("/s3"), Ok(Target::Service)); + assert_eq!(parse("/s3/"), Ok(Target::Service)); + assert_eq!(parse("/s3/sccache"), Ok(Target::Bucket("sccache".into()))); + assert_eq!(parse("/s3/sccache/"), Ok(Target::Bucket("sccache".into()))); + assert_eq!(parse("/s3/a%2Db"), Ok(Target::Bucket("a-b".into()))); + for bad in [ + "/s3//", + "/s3//k", + "/s3/a%2Fb", + "/s3/..", + "/s3/.", + "/s3/tags", + "/s3/blobs/k", + "/s3/a b", + "/s3/a%20b", + "/s3/%C3%A9", + "/s3/a%2", + ] { + let r = parse(bad).unwrap_err(); + assert_eq!(r.code, "InvalidBucketName", "{bad}"); + } + } + + #[test] + fn sccache_keys_map_onto_the_files_tree() { + assert_eq!( + object(".sccache_check"), + Ok(Target::Object { + key: ".sccache_check".into(), + segs: vec!["b".into(), ".sccache_check".into()], + }) + ); + let hex = "0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0"; + let Ok(Target::Object { key, segs }) = object(&format!("a/b/c/{hex}")) else { + panic!("a sharded key"); + }; + assert_eq!(key, format!("a/b/c/{hex}")); + assert_eq!(segs, ["b", "a", "b", "c", hex]); + let (repo, tag) = crate::files::object_of(&segs).unwrap(); + assert_eq!((repo.as_str(), tag), ("files/b/a/b/c", hex)); + assert!(crate::valid_name(&crate::files::repo_name(&segs))); + // An escape of an allowed character is that character. + assert!(matches!(object("a%2Db"), Ok(Target::Object { key, .. }) if key == "a-b")); + } + + #[test] + fn the_deepest_and_longest_keys_pass() { + let deepest = vec!["x"; MAX_KEY_COMPONENTS].join("/"); + let Ok(Target::Object { segs, .. }) = object(&deepest) else { + panic!("14 components"); + }; + assert_eq!(segs.len(), crate::MAX_NAME_SEGMENTS - 1); + assert!(crate::valid_name(&crate::files::repo_name(&segs))); + assert!(object(&"x".repeat(255)).is_ok()); + assert_eq!(longest().len(), MAX_KEY); + assert!(object(&longest()).is_ok()); + } + + /// A key of exactly [`MAX_KEY`] bytes, no component over 255. + fn longest() -> String { + format!("{0}/{0}/{0}/{1}/y", "x".repeat(255), "x".repeat(254)) + } + + #[test] + fn keys_outside_the_store_s_names_are_refused_naming_the_rule() { + let too_deep = vec!["x"; MAX_KEY_COMPONENTS + 1].join("/"); + assert!(refused(&too_deep, "InvalidArgument").contains("14")); + assert!(refused(&"x".repeat(256), "KeyTooLongError").contains("255")); + let over = format!("{}y", longest()); + assert_eq!(over.len(), MAX_KEY + 1); + assert!(refused(&over, "KeyTooLongError").contains("1024")); + for empty in ["a//b", "a/", "a/b/", "/a"] { + assert!( + refused(empty, "InvalidArgument").contains("empty"), + "{empty}" + ); + } + for dot in [".", "..", "a/./b", "a/../b", "%2E%2E/x"] { + assert!(refused(dot, "InvalidArgument").contains(". or .."), "{dot}"); + } + for reserved in ["tags", "a/manifests/b", "blobs/x", "a/blobs"] { + assert!( + refused(reserved, "InvalidArgument").contains("tags, manifests or blobs"), + "{reserved}" + ); + } + for bad in [ + "a%2Fb", + "a%5Cb", + "%C3%A9", + "caf\u{e9}", + "a b", + "a%20b", + "a%00b", + "a%0Ab", + "a%7F", + "a+b", + "a~b", + "a:b", + "a*b", + ] { + assert!( + refused(bad, "InvalidArgument").contains("A-Z a-z 0-9 . _ -"), + "{bad}" + ); + } + for bad in ["a%2", "a%G0", "%FF", "%C3"] { + assert!(refused(bad, "InvalidArgument").contains("escape"), "{bad}"); + } + } +} diff --git a/vk-registry/src/s3/xml.rs b/vk-registry/src/s3/xml.rs index 6217ecf7..721e126c 100644 --- a/vk-registry/src/s3/xml.rs +++ b/vk-registry/src/s3/xml.rs @@ -1,11 +1,39 @@ -//! The XML S3 answers in: the `Error` document every refusal carries. +//! The XML S3 answers in: the `Error` document every refusal carries, and the documents of +//! the operations that answer with one. use bytes::Bytes; -use hyper::header::CONTENT_TYPE; +use hyper::header::{CONTENT_LENGTH, CONTENT_TYPE}; use hyper::{Response, StatusCode}; use crate::{Body, body_of}; +/// The namespace of S3's documents. +pub(crate) const NS: &str = "http://s3.amazonaws.com/doc/2006-03-01/"; + +/// The XML declaration every document starts with. +const DECLARATION: &str = ""; + +/// A 200 answer whose document is `root`, in S3's namespace, holding `inner`: elements +/// already built, every caller string in them escaped. +pub(crate) fn document(root: &str, inner: &str) -> Response { + let body = if inner.is_empty() { + format!("{DECLARATION}<{root} xmlns=\"{NS}\"/>") + } else { + format!("{DECLARATION}<{root} xmlns=\"{NS}\">{inner}") + }; + Response::builder() + .status(StatusCode::OK) + .header(CONTENT_TYPE, "application/xml") + .header(CONTENT_LENGTH, body.len().to_string()) + .body(body_of(Bytes::from(body))) + .expect("building an S3 document") +} + +/// `text`, the text escaped. +pub(crate) fn element(name: &str, text: &str) -> String { + format!("<{name}>{}", escape(text)) +} + /// Escape a string for XML text: [`crate::html_escape`]'s five characters are XML's too. /// What reaches it is a request path, a header value or a message of ours, none of which /// can hold a control character XML 1.0 has no way to write. @@ -25,12 +53,12 @@ pub(crate) fn error( request_id: &str, ) -> Response { let mut body = format!( - "{}{}", + "{DECLARATION}{}{}", escape(code), escape(message) ); for (name, value) in extra { - body.push_str(&format!("<{name}>{}", escape(value))); + body.push_str(&element(name, value)); } body.push_str(&format!( "{}{}", @@ -71,4 +99,30 @@ mod tests { 0123456789abcdef" ); } + + #[tokio::test] + async fn a_document_is_in_s3_s_namespace() { + let body = async |resp: Response| { + assert_eq!(resp.status(), 200); + assert_eq!(resp.headers()[CONTENT_TYPE], "application/xml"); + let len: usize = resp.headers()[CONTENT_LENGTH] + .to_str() + .unwrap() + .parse() + .unwrap(); + let body = resp.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(body.len(), len); + String::from_utf8(body.to_vec()).unwrap() + }; + assert_eq!( + body(document("LocationConstraint", "")).await, + "\ + " + ); + assert_eq!( + body(document("R", &element("Name", "a\ + a<b" + ); + } } diff --git a/vk-registry/tests/s3_e2e.rs b/vk-registry/tests/s3_e2e.rs index b2ea66ef..12879658 100644 --- a/vk-registry/tests/s3_e2e.rs +++ b/vk-registry/tests/s3_e2e.rs @@ -1,7 +1,8 @@ //! S3 over real HTTP: the gate `/s3/` requests pass, under every authentication mode the -//! server has. SigV4 requests are signed here by a signer of the tests' own, checked against -//! AWS's published example; responses are read off a raw socket, so a test sees exactly what -//! a connection carries and can send a second request on it. +//! server has, and the operations on buckets and objects behind it. SigV4 requests are signed +//! here by a signer of the tests' own, checked against AWS's published example; responses are +//! read off a raw socket, so a test sees exactly what a connection carries and can send a +//! second request on it. use std::sync::Arc; use std::time::{Duration, SystemTime}; @@ -251,6 +252,11 @@ impl Req { /// Signed for `host` by `key` (id, secret), `skew` seconds off now (UNSIGNED-PAYLOAD). fn signed(self, host: &str, key: (&str, &str), skew: i64) -> Self { + self.signed_payload(host, key, skew, UNSIGNED) + } + + /// [`Req::signed`] with `payload` as the `x-amz-content-sha256` the signature binds. + fn signed_payload(self, host: &str, key: (&str, &str), skew: i64, payload: &str) -> Self { let at = stamp(vk_peer::now_secs().saturating_add_signed(skew)); let auth = authorization( self.method, @@ -258,14 +264,14 @@ impl Req { &self.query, &[ ("host", host), - ("x-amz-content-sha256", UNSIGNED), + ("x-amz-content-sha256", payload), ("x-amz-date", &at), ], - UNSIGNED, + payload, key, &at, ); - self.header("x-amz-content-sha256", UNSIGNED) + self.header("x-amz-content-sha256", payload) .header("x-amz-date", &at) .header("Authorization", &auth) } @@ -285,12 +291,13 @@ impl Req { } } -/// A response: its status, headers (names lowercase) and body. +/// A response: its status, headers (names lowercase) and body, also as text. #[derive(Debug)] struct Resp { status: u16, headers: Vec<(String, String)>, body: String, + bytes: Vec, } impl Resp { @@ -301,10 +308,9 @@ impl Resp { .map(|(_, v)| v.as_str()) } - /// The status, the S3 headers every `/s3/` answer carries, and for an error its XML - /// `Code` and the request id the header names. + /// The status and the S3 headers every `/s3/` answer carries. #[track_caller] - fn is(&self, status: u16, code: &str) -> &Self { + fn ok(&self, status: u16) -> &Self { assert_eq!(self.status, status, "{self:?}"); let id = self.header("x-amz-request-id").expect("a request id"); assert!( @@ -312,6 +318,14 @@ impl Resp { "{id}" ); assert_eq!(self.header("x-content-type-options"), Some("nosniff")); + self + } + + /// [`Resp::ok`], and for an error its XML `Code` and the request id the header names. + #[track_caller] + fn is(&self, status: u16, code: &str) -> &Self { + self.ok(status); + let id = self.header("x-amz-request-id").expect("a request id"); assert_eq!(self.header("content-type"), Some("application/xml")); assert!( self.body @@ -359,10 +373,16 @@ impl Conn { .await .unwrap(); write.write_all(&req.body).await.unwrap(); - self.response().await + self.response_to(req.method == "HEAD").await } async fn response(&mut self) -> Resp { + self.response_to(false).await + } + + /// The next response; `head` when it answers a HEAD, which has no body whatever its + /// `Content-Length` says. + async fn response_to(&mut self, head: bool) -> Resp { let mut line = String::new(); self.read.read_line(&mut line).await.unwrap(); let status = line @@ -384,13 +404,15 @@ impl Conn { let len: usize = headers .iter() .find(|(n, _)| n == "content-length") + .filter(|_| !head) .map_or(0, |(_, v)| v.parse().unwrap()); - let mut body = vec![0; len]; - self.read.read_exact(&mut body).await.unwrap(); + let mut bytes = vec![0; len]; + self.read.read_exact(&mut bytes).await.unwrap(); Resp { status, headers, - body: String::from_utf8(body).unwrap(), + body: String::from_utf8_lossy(&bytes).into_owned(), + bytes, } } @@ -486,20 +508,24 @@ async fn an_open_server_checks_no_signature_but_refuses_presigned_and_tokens() { send(&host, get("/s3/b/k/deeper")) .await .is(404, "NoSuchKey"); - send(&host, get("/s3/b")).await.is(404, "NoSuchBucket"); - send(&host, get("/s3/b/")).await.is(404, "NoSuchBucket"); - send(&host, get("/s3/")).await.is(501, "NotImplemented"); - send(&host, get("/s3")).await.is(501, "NotImplemented"); + // Listing a bucket is not served yet; listing the buckets is not on an open server. + send(&host, get("/s3/b")).await.is(501, "NotImplemented"); + send(&host, get("/s3/b/")).await.is(501, "NotImplemented"); + send(&host, get("/s3/")).await.is(403, "AccessDenied"); + send(&host, get("/s3")).await.is(403, "AccessDenied"); send( &host, Req::new("PUT", "/s3/b/k").body(b"Hello, World!".to_vec()), ) .await - .is(501, "NotImplemented"); + .ok(200); + let resp = send(&host, get("/s3/b/k")).await; + resp.ok(200); + assert_eq!(resp.body, "Hello, World!"); // Any key will do, its signature unchecked. send( &host, - get("/s3/b/k").signed(&host, ("anyone", "anything"), 0), + get("/s3/b/absent").signed(&host, ("anyone", "anything"), 0), ) .await .is(404, "NoSuchKey"); @@ -546,20 +572,24 @@ async fn a_shared_bearer_is_the_access_key_and_its_secret() { .is(404, "NoSuchKey"); sigv4_refusals(&host, ("s3cret", "s3cret")).await; - // A hub token is a key too, signed or as a bearer; an expired one, or one for another - // registry, is not. + // A hub token is a key too, signed or as a bearer, for what its grants cover; an expired + // one, or one for another registry, is not. let now = vk_peer::now_secs(); let token = signer.sign(hub_claims(AUD, now - 1, now + 600)).unwrap(); - send(&host, get().signed(&host, (&token, &token), 0)) + let granted = || Req::new("GET", "/s3/p/gl/42/k"); + send(&host, granted().signed(&host, (&token, &token), 0)) .await .is(404, "NoSuchKey"); send( &host, - get().header("Authorization", &format!("Bearer {token}")), + granted().header("Authorization", &format!("Bearer {token}")), ) .await .is(404, "NoSuchKey"); - send(&host, get().signed(&host, (&token, "other"), 0)) + send(&host, get().signed(&host, (&token, &token), 0)) + .await + .is(403, "AccessDenied"); + send(&host, granted().signed(&host, (&token, "other"), 0)) .await .is(403, "SignatureDoesNotMatch"); for bad in [ @@ -807,3 +837,736 @@ async fn a_read_only_registry_refuses_writes_with_retry_after() { .await .is(404, "NoSuchKey"); } + +// ---- objects and buckets ---- + +const HELLO: &[u8] = b"Hello, World!"; +const MANIFEST_TYPE: &str = "application/vnd.oci.image.manifest.v1+json"; + +fn md5_hex(b: &[u8]) -> String { + hex(&md5::compute(b).0) +} + +fn sha256_hex(b: &[u8]) -> String { + hex(&sha2::Sha256::digest(b)) +} + +fn b64(b: &[u8]) -> String { + vk_hub_proto::to_base64(b) +} + +/// Files in `uploads/`, where a PUT stages its object. +fn staging_left(dir: &std::path::Path) -> usize { + std::fs::read_dir(dir.join("store/uploads")).map_or(0, |d| { + d.filter_map(|e| e.ok()) + .filter(|e| e.file_type().is_ok_and(|t| t.is_file())) + .count() + }) +} + +/// How a test's requests are authenticated: a function from a request to the signed one. +type Sign<'a> = &'a dyn Fn(Req) -> Req; + +/// What every served object carries, beside its bytes. +#[track_caller] +fn is_object(resp: &Resp, body: &[u8]) { + resp.ok(200); + assert_eq!(resp.bytes, body); + assert_eq!( + resp.header("content-length"), + Some(&*body.len().to_string()) + ); + assert_eq!( + resp.header("content-type"), + Some("application/octet-stream") + ); + assert_eq!(resp.header("content-disposition"), Some("attachment")); + assert!(resp.header("last-modified").is_some(), "{resp:?}"); + assert_eq!(resp.header("accept-ranges"), None); + assert_eq!(resp.header("docker-content-digest"), None); +} + +/// What `sccache` does with its S3 backend, OpenDAL's requests: its startup probe of +/// `.sccache_check`, the probe written, a cache entry written and read back, and the HEAD +/// OpenDAL's `stat` sends. +async fn sccache_sequence(host: &str, sign: Sign<'_>, prefix: &str) { + let check = format!("/s3/sccache/{prefix}.sccache_check"); + send(host, sign(Req::new("GET", &check))) + .await + .is(404, "NoSuchKey"); + let resp = send(host, sign(Req::new("PUT", &check).body(HELLO.to_vec()))).await; + resp.ok(200); + let etag = format!("\"{}\"", md5_hex(HELLO)); + assert_eq!(resp.header("etag"), Some(&*etag)); + let resp = send(host, sign(Req::new("GET", &check))).await; + is_object(&resp, HELLO); + assert_eq!(resp.header("etag"), Some(&*etag)); + + let entry: Vec = (0..70_000u32).map(|i| (i % 251) as u8).collect(); + let key = format!( + "/s3/sccache/{prefix}a/b/c/{}", + sha256_hex(b"an entry's key") + ); + send(host, sign(Req::new("PUT", &key).body(entry.clone()))) + .await + .ok(200); + let resp = send(host, sign(Req::new("GET", &key))).await; + is_object(&resp, &entry); + assert_eq!( + resp.header("etag"), + Some(&*format!("\"{}\"", md5_hex(&entry))) + ); + let resp = send(host, sign(Req::new("HEAD", &key))).await; + resp.ok(200); + assert!(resp.bytes.is_empty()); + assert_eq!( + resp.header("content-length"), + Some(&*entry.len().to_string()) + ); + assert_eq!(resp.header("content-disposition"), Some("attachment")); +} + +#[tokio::test] +async fn the_sccache_sequence_round_trips_on_an_open_server() { + let dir = tmp("sccache-open"); + let host = spawn(open_state(&dir, true)); + sccache_sequence(&host, &|r| r, "").await; + let signed = |r: Req| r.signed(&host, ("anyone", "anything"), 0); + sccache_sequence(&host, &signed, "ci/").await; + assert_eq!(staging_left(&dir), 0); +} + +#[tokio::test] +async fn the_sccache_sequence_round_trips_with_an_api_key() { + let dir = tmp("sccache-accounts"); + let state = accounts_state(&dir); + let db = accounts_db(&state); + let user = db + .upsert_user("https://issuer", "ci", EmailUpdate::Keep, None) + .unwrap(); + let scopes = [Scope { + action: Action::Write, + repo_pattern: "files/*".to_string(), + }]; + let (_, key) = db + .create_api_key(Some(&user.id), "s3", &scopes, None) + .unwrap(); + let host = spawn(state.clone()); + let signed = |r: Req| r.signed(&host, (&key, &key), 0); + sccache_sequence(&host, &signed, "ci/").await; + // The bearer on the same paths still works. + let bearer = |r: Req| r.header("Authorization", &format!("Bearer {key}")); + sccache_sequence(&host, &bearer, "other/").await; +} + +/// The payload modes: unsigned, or the body's SHA-256, which the body must then have. A +/// streamed `aws-chunked` body is not decoded yet, so it is refused and its framing never +/// becomes an object. +#[tokio::test] +async fn a_put_body_is_unsigned_or_its_signed_sha256_and_streaming_waits() { + let dir = tmp("payload"); + let (state, _) = shared( + &dir, + Auth::Bearer { + token: "s3cret".into(), + }, + true, + ); + let host = spawn(state); + let key = ("s3cret", "s3cret"); + let put = |path: &str, body: &[u8]| Req::new("PUT", path).body(body.to_vec()); + let get = |path: &str| Req::new("GET", path).signed(&host, key, 0); + + send(&host, put("/s3/b/unsigned", HELLO).signed(&host, key, 0)) + .await + .ok(200); + is_object(&send(&host, get("/s3/b/unsigned")).await, HELLO); + + // The body's own digest, any case. + let hex = sha256_hex(HELLO); + for (path, digest) in [ + ("/s3/b/hex", hex.clone()), + ("/s3/b/HEX", hex.to_uppercase()), + ] { + let req = put(path, HELLO).signed_payload(&host, key, 0, &digest); + send(&host, req).await.ok(200); + is_object(&send(&host, get(path)).await, HELLO); + } + // Another body's: refused, and nothing tagged. + let req = put("/s3/b/mismatch", b"Hello, World?").signed_payload(&host, key, 0, &hex); + send(&host, req).await.is(400, "XAmzContentSHA256Mismatch"); + send(&host, get("/s3/b/mismatch")) + .await + .is(404, "NoSuchKey"); + let req = put("/s3/b/garbage", HELLO).signed_payload(&host, key, 0, "not-a-digest"); + send(&host, req).await.is(400, "InvalidArgument"); + + // AWS's chunked framing, signed or announced: not stored as the object it would become. + let framed = b"d;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648\r\n\ +Hello, World!\r\n0;chunk-signature=b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\r\n\r\n"; + for (path, req) in [ + ( + "/s3/b/streaming", + put("/s3/b/streaming", framed).signed_payload( + &host, + key, + 0, + "STREAMING-AWS4-HMAC-SHA256-PAYLOAD", + ), + ), + ( + "/s3/b/trailer", + put("/s3/b/trailer", framed).signed_payload( + &host, + key, + 0, + "STREAMING-UNSIGNED-PAYLOAD-TRAILER", + ), + ), + ( + "/s3/b/chunked", + put("/s3/b/chunked", framed) + .header("Content-Encoding", "aws-chunked") + .header("x-amz-decoded-content-length", "13") + .signed(&host, key, 0), + ), + ] { + send(&host, req).await.is(501, "NotImplemented"); + send(&host, get(path)).await.is(404, "NoSuchKey"); + } + assert!( + !dir.join("store/repos/files/b/tags/streaming").exists() + && !dir.join("store/repos/files/b/tags/chunked").exists() + ); + assert_eq!(staging_left(&dir), 0); + + // A declared length past 4 GiB is refused before a byte is read. + let req = put("/s3/b/big", b"").signed(&host, key, 0); + let head = req.head(&host).replace( + "Content-Length: 0", + &format!("Content-Length: {}", (4u64 << 30) + 1), + ); + let mut conn = Conn::open(&host).await; + conn.write + .as_mut() + .unwrap() + .write_all(head.as_bytes()) + .await + .unwrap(); + let resp = tokio::time::timeout(Duration::from_secs(10), conn.response()) + .await + .expect("answered unread"); + resp.is(413, "EntityTooLarge"); + assert_eq!(staging_left(&dir), 0); +} + +#[tokio::test] +async fn a_put_checks_the_digests_it_is_sent() { + let dir = tmp("digests"); + let host = spawn(open_state(&dir, true)); + let put = |path: &str, (name, value): (&str, &str)| { + Req::new("PUT", path) + .header(name, value) + .body(HELLO.to_vec()) + }; + let get = |path: &str| Req::new("GET", path); + let other = b"Hello, World?"; + let sha1 = |b: &[u8]| { + use sha1::Digest as _; + sha1::Sha1::digest(b).to_vec() + }; + let crc = |b: &[u8]| crc32fast::hash(b).to_be_bytes().to_vec(); + let good: Vec<(&str, String)> = vec![ + ("Content-MD5", b64(&md5::compute(HELLO).0)), + ("x-amz-checksum-sha256", b64(&sha2::Sha256::digest(HELLO))), + ("x-amz-checksum-sha1", b64(&sha1(HELLO))), + ("x-amz-checksum-crc32", b64(&crc(HELLO))), + ]; + let bad: Vec<(&str, String)> = vec![ + ("Content-MD5", b64(&md5::compute(other).0)), + ("x-amz-checksum-sha256", b64(&sha2::Sha256::digest(other))), + ("x-amz-checksum-sha1", b64(&sha1(other))), + ("x-amz-checksum-crc32", b64(&crc(other))), + ]; + for (i, (name, value)) in good.iter().enumerate() { + let path = format!("/s3/b/good{i}"); + let resp = send(&host, put(&path, (name, value))).await; + resp.ok(200); + if name.starts_with("x-amz-checksum") { + assert_eq!(resp.header(name), Some(value.as_str()), "{name}"); + } + is_object(&send(&host, get(&path)).await, HELLO); + } + for (i, (name, value)) in bad.iter().enumerate() { + let path = format!("/s3/b/bad{i}"); + send(&host, put(&path, (name, value))) + .await + .is(400, "BadDigest"); + send(&host, get(&path)).await.is(404, "NoSuchKey"); + } + send(&host, put("/s3/b/x", ("Content-MD5", "AAAA"))) + .await + .is(400, "InvalidDigest"); + send(&host, put("/s3/b/x", ("x-amz-checksum-sha256", "AAAA"))) + .await + .is(400, "InvalidRequest"); + // Not checked, so taken. + send(&host, put("/s3/b/c", ("x-amz-checksum-crc32c", "AAAAAA=="))) + .await + .ok(200); + assert_eq!(staging_left(&dir), 0); + + // On a read, the SHA-256 the store has, when asked. + let resp = send(&host, get("/s3/b/good0")).await; + assert_eq!(resp.header("x-amz-checksum-sha256"), None); + let asked = get("/s3/b/good0").header("x-amz-checksum-mode", "ENABLED"); + let resp = send(&host, asked).await; + is_object(&resp, HELLO); + assert_eq!( + resp.header("x-amz-checksum-sha256"), + Some(&*b64(&sha2::Sha256::digest(HELLO))) + ); + let asked = Req::new("HEAD", "/s3/b/good0").header("x-amz-checksum-mode", "enabled"); + let resp = send(&host, asked).await; + resp.ok(200); + assert_eq!( + resp.header("x-amz-checksum-sha256"), + Some(&*b64(&sha2::Sha256::digest(HELLO))) + ); +} + +#[tokio::test] +async fn user_metadata_round_trips_and_no_content_type_is_kept() { + let dir = tmp("meta"); + let host = spawn(open_state(&dir, true)); + let resp = send( + &host, + Req::new("PUT", "/s3/b/m") + .header("Content-Type", "text/html") + .header("x-amz-meta-mtime", "1700000000") + .header("X-Amz-Meta-Foo-Bar", "baz qux") + .body(b"