From 27b127c3089baa922032046ff520452de79cbd63 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Mon, 25 Apr 2022 18:34:37 +0200 Subject: [PATCH 01/12] Added mod bucket --- src/commons/bucket.rs | 64 +++++++++++++++++++++++++++++++++++++++++++ src/commons/mod.rs | 1 + 2 files changed, 65 insertions(+) create mode 100644 src/commons/bucket.rs diff --git a/src/commons/bucket.rs b/src/commons/bucket.rs new file mode 100644 index 000000000..0adf2c751 --- /dev/null +++ b/src/commons/bucket.rs @@ -0,0 +1,64 @@ +//! Implementation of the bucket definition as described in +//! https://github.com/stackabletech/documentation/pull/177 +//! +//! +use crate::commons::tls::Tls; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct S3Bucket { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bucket_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub connection: Option, +} + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum ConnectionDef { + Inline(S3Connection), + Reference(String), +} +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct S3Connection { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub host: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub port: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub secret_class: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tls: Option, +} + +#[cfg(test)] +mod test { + use crate::commons::bucket::ConnectionDef::Inline; + use crate::commons::bucket::{S3Bucket, S3Connection}; + + #[test] + fn test_ser_inline() { + let bucket = S3Bucket { + bucket_name: Some("test-bucket-name".to_owned()), + connection: Some(Inline(S3Connection { + host: Some("host".to_owned()), + port: Some(8080), + secret_class: None, + tls: None, + })), + }; + + assert_eq!( + serde_yaml::to_string(&bucket).unwrap(), + "--- +bucketName: test-bucket-name +connection: + inline: + host: host + port: 8080 +".to_owned()) + } +} diff --git a/src/commons/mod.rs b/src/commons/mod.rs index ac2ad7132..80ac61b64 100644 --- a/src/commons/mod.rs +++ b/src/commons/mod.rs @@ -1,6 +1,7 @@ //! This module provides common datastructures or CRDs shared between all the operators pub mod authentication; +pub mod bucket; pub mod ldap; pub mod opa; pub mod resources; From 2505008372dc65be4d256eb13a286969ec37fe30 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 26 Apr 2022 17:37:13 +0200 Subject: [PATCH 02/12] Experiment with S3 structs. --- src/commons/bucket.rs | 64 ------------------ src/commons/mod.rs | 2 +- src/commons/s3.rs | 152 ++++++++++++++++++++++++++++++++++++++++++ src/error.rs | 11 +++ 4 files changed, 164 insertions(+), 65 deletions(-) delete mode 100644 src/commons/bucket.rs create mode 100644 src/commons/s3.rs diff --git a/src/commons/bucket.rs b/src/commons/bucket.rs deleted file mode 100644 index 0adf2c751..000000000 --- a/src/commons/bucket.rs +++ /dev/null @@ -1,64 +0,0 @@ -//! Implementation of the bucket definition as described in -//! https://github.com/stackabletech/documentation/pull/177 -//! -//! -use crate::commons::tls::Tls; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct S3Bucket { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub bucket_name: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub connection: Option, -} - -#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub enum ConnectionDef { - Inline(S3Connection), - Reference(String), -} -#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct S3Connection { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub host: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub port: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub secret_class: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub tls: Option, -} - -#[cfg(test)] -mod test { - use crate::commons::bucket::ConnectionDef::Inline; - use crate::commons::bucket::{S3Bucket, S3Connection}; - - #[test] - fn test_ser_inline() { - let bucket = S3Bucket { - bucket_name: Some("test-bucket-name".to_owned()), - connection: Some(Inline(S3Connection { - host: Some("host".to_owned()), - port: Some(8080), - secret_class: None, - tls: None, - })), - }; - - assert_eq!( - serde_yaml::to_string(&bucket).unwrap(), - "--- -bucketName: test-bucket-name -connection: - inline: - host: host - port: 8080 -".to_owned()) - } -} diff --git a/src/commons/mod.rs b/src/commons/mod.rs index 80ac61b64..41bdcbbcb 100644 --- a/src/commons/mod.rs +++ b/src/commons/mod.rs @@ -1,9 +1,9 @@ //! This module provides common datastructures or CRDs shared between all the operators pub mod authentication; -pub mod bucket; pub mod ldap; pub mod opa; pub mod resources; +pub mod s3; pub mod secret_class; pub mod tls; diff --git a/src/commons/s3.rs b/src/commons/s3.rs new file mode 100644 index 000000000..5bbc721e7 --- /dev/null +++ b/src/commons/s3.rs @@ -0,0 +1,152 @@ +//! Implementation of the bucket definition as described in +//! https://github.com/stackabletech/documentation/pull/177 +//! +//! +use crate::commons::tls::Tls; +use crate::error; +use crate::{client::Client, error::OperatorResult}; +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, CustomResource, Debug, Default, Deserialize, JsonSchema, Eq, PartialEq, Serialize)] +#[kube( + group = "s3.stackable.tech", + version = "v1alpha1", + kind = "S3Bucket", + plural = "s3buckets", + crates( + kube_core = "kube::core", + k8s_openapi = "k8s_openapi", + schemars = "schemars" + ), + namespaced +)]#[serde(rename_all = "camelCase")] +pub struct S3BucketSpec { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bucket_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub connection: Option, +} + +impl S3Bucket { + pub async fn get( + resource_name: &str, + client: &Client, + namespace: Option<&str>, + ) -> OperatorResult { + client + .get::(resource_name, namespace) + .await + .map_err(|_source| error::Error::MissingS3Bucket { + name: resource_name.to_string(), + }) + } + + pub async fn secret_class(&self, client: &Client, namespace: Option<&str>) -> Option { + match self.spec.connection.as_ref() { + Some(ConnectionDef::Inline(S3ConnectionSpec { secret_class, .. })) => { + secret_class.clone() + } + Some(ConnectionDef::Reference(s3_conn_ref)) => { + S3Connection::get(s3_conn_ref.as_ref(), client, namespace) + .await + .map_or(None,|s3c| s3c.spec.secret_class) + } + _ => None, + } + } +} + +#[derive(Clone, Debug, Deserialize, JsonSchema, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum S3BucketDef { + Inline(S3BucketSpec), + Reference(String), +} + +impl S3BucketDef { + pub async fn secret_class(&self, client: &Client, namespace: Option<&str>) -> Option { + match self { + S3BucketDef::Inline(s3_bucket) => s3_bucket.secret_class(client, namespace).await, + S3BucketDef::Reference(_s3_bucket) => todo!("get secret_class from bucket ref"), + } + } +} + +#[derive(Clone, Debug, Deserialize, JsonSchema, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum ConnectionDef { + Inline(S3ConnectionSpec), + Reference(String), +} + +#[derive(CustomResource, Clone, Debug, Default, Deserialize, JsonSchema, Eq, PartialEq, Serialize)] +#[kube( + group = "s3.stackable.tech", + version = "v1alpha1", + kind = "S3Connection", + plural = "s3connections", + crates( + kube_core = "kube::core", + k8s_openapi = "k8s_openapi", + schemars = "schemars" + ), + namespaced +)] +#[serde(rename_all = "camelCase")] +pub struct S3ConnectionSpec { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub host: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub port: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub secret_class: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tls: Option, +} +impl S3Connection { + pub async fn get( + resource_name: &str, + client: &Client, + namespace: Option<&str>, + ) -> OperatorResult { + client + .get::(resource_name, namespace) + .await + .map_err(|_source| error::Error::MissingS3Connection { + name: resource_name.to_string(), + }) + } +} + +#[cfg(test)] +mod test { + use crate::commons::s3::ConnectionDef::Inline; + use crate::commons::s3::{S3BucketSpec, S3ConnectionSpec}; + + #[test] + fn test_ser_inline() { + let bucket = S3BucketSpec { + bucket_name: Some("test-bucket-name".to_owned()), + connection: Some(Inline(S3ConnectionSpec { + host: Some("host".to_owned()), + port: Some(8080), + secret_class: None, + tls: None, + })), + }; + + assert_eq!( + serde_yaml::to_string(&bucket).unwrap(), + "--- +bucketName: test-bucket-name +connection: + inline: + host: host + port: 8080 +" + .to_owned() + ) + } +} diff --git a/src/error.rs b/src/error.rs index f878c43e7..40aefb04e 100644 --- a/src/error.rs +++ b/src/error.rs @@ -64,6 +64,17 @@ pub enum Error { #[error("Missing OPA connect string in configmap [{configmap_name}]")] MissingOpaConnectString { configmap_name: String }, + + #[error("Missing S3 connection [{name}]")] + MissingS3Connection { + name: String, + }, + + #[error("Missing S3 bucket [{name}]")] + MissingS3Bucket { + name: String, + }, + } pub type OperatorResult = std::result::Result; From 7c3c5aaf52ce903ad88c1c59d6dea60298bdcedc Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 26 Apr 2022 18:14:37 +0200 Subject: [PATCH 03/12] It compiles. --- src/commons/s3.rs | 29 +++++++++++++++-------------- src/error.rs | 9 ++------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/commons/s3.rs b/src/commons/s3.rs index 5bbc721e7..1411a904c 100644 --- a/src/commons/s3.rs +++ b/src/commons/s3.rs @@ -9,7 +9,7 @@ use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -#[derive(Clone, CustomResource, Debug, Default, Deserialize, JsonSchema, Eq, PartialEq, Serialize)] +#[derive(Clone, CustomResource, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] #[kube( group = "s3.stackable.tech", version = "v1alpha1", @@ -21,7 +21,8 @@ use serde::{Deserialize, Serialize}; schemars = "schemars" ), namespaced -)]#[serde(rename_all = "camelCase")] +)] +#[serde(rename_all = "camelCase")] pub struct S3BucketSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub bucket_name: Option, @@ -29,36 +30,36 @@ pub struct S3BucketSpec { pub connection: Option, } -impl S3Bucket { +impl S3BucketSpec { pub async fn get( resource_name: &str, client: &Client, namespace: Option<&str>, - ) -> OperatorResult { + ) -> OperatorResult { client - .get::(resource_name, namespace) + .get::(resource_name, namespace) .await .map_err(|_source| error::Error::MissingS3Bucket { name: resource_name.to_string(), }) } - pub async fn secret_class(&self, client: &Client, namespace: Option<&str>) -> Option { - match self.spec.connection.as_ref() { + pub async fn secret_class(&self, client: &Client, namespace: Option) -> Option { + match self.connection.as_ref() { Some(ConnectionDef::Inline(S3ConnectionSpec { secret_class, .. })) => { secret_class.clone() } Some(ConnectionDef::Reference(s3_conn_ref)) => { S3Connection::get(s3_conn_ref.as_ref(), client, namespace) .await - .map_or(None,|s3c| s3c.spec.secret_class) + .map_or(None, |s3c| s3c.spec.secret_class) } _ => None, } } } -#[derive(Clone, Debug, Deserialize, JsonSchema, Eq, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub enum S3BucketDef { Inline(S3BucketSpec), @@ -66,7 +67,7 @@ pub enum S3BucketDef { } impl S3BucketDef { - pub async fn secret_class(&self, client: &Client, namespace: Option<&str>) -> Option { + pub async fn secret_class(&self, client: &Client, namespace: Option) -> Option { match self { S3BucketDef::Inline(s3_bucket) => s3_bucket.secret_class(client, namespace).await, S3BucketDef::Reference(_s3_bucket) => todo!("get secret_class from bucket ref"), @@ -74,14 +75,14 @@ impl S3BucketDef { } } -#[derive(Clone, Debug, Deserialize, JsonSchema, Eq, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub enum ConnectionDef { Inline(S3ConnectionSpec), Reference(String), } -#[derive(CustomResource, Clone, Debug, Default, Deserialize, JsonSchema, Eq, PartialEq, Serialize)] +#[derive(CustomResource, Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] #[kube( group = "s3.stackable.tech", version = "v1alpha1", @@ -109,10 +110,10 @@ impl S3Connection { pub async fn get( resource_name: &str, client: &Client, - namespace: Option<&str>, + namespace: Option, ) -> OperatorResult { client - .get::(resource_name, namespace) + .get::(resource_name, namespace.as_deref()) .await .map_err(|_source| error::Error::MissingS3Connection { name: resource_name.to_string(), diff --git a/src/error.rs b/src/error.rs index 40aefb04e..777449f7b 100644 --- a/src/error.rs +++ b/src/error.rs @@ -66,15 +66,10 @@ pub enum Error { MissingOpaConnectString { configmap_name: String }, #[error("Missing S3 connection [{name}]")] - MissingS3Connection { - name: String, - }, + MissingS3Connection { name: String }, #[error("Missing S3 bucket [{name}]")] - MissingS3Bucket { - name: String, - }, - + MissingS3Bucket { name: String }, } pub type OperatorResult = std::result::Result; From fa99aeaeda2ce453849342a9dbc5845df9f224bc Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 27 Apr 2022 17:57:37 +0200 Subject: [PATCH 04/12] Implement S3BucketDef::resolve. --- src/commons/s3.rs | 53 ++++++++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/src/commons/s3.rs b/src/commons/s3.rs index 1411a904c..baeea097d 100644 --- a/src/commons/s3.rs +++ b/src/commons/s3.rs @@ -1,6 +1,7 @@ //! Implementation of the bucket definition as described in //! https://github.com/stackabletech/documentation/pull/177 //! +//! Operator CRDs are expected to use the [`S3BucketDef`] as an entry point to this module, //! use crate::commons::tls::Tls; use crate::error; @@ -34,27 +35,30 @@ impl S3BucketSpec { pub async fn get( resource_name: &str, client: &Client, - namespace: Option<&str>, - ) -> OperatorResult { + namespace: Option, + ) -> OperatorResult { client - .get::(resource_name, namespace) + .get::(resource_name, namespace.as_deref()) .await + .map(|crd| crd.spec) .map_err(|_source| error::Error::MissingS3Bucket { name: resource_name.to_string(), }) } - pub async fn secret_class(&self, client: &Client, namespace: Option) -> Option { + pub async fn inlined( + &self, + client: &Client, + namespace: Option, + ) -> OperatorResult { match self.connection.as_ref() { - Some(ConnectionDef::Inline(S3ConnectionSpec { secret_class, .. })) => { - secret_class.clone() - } - Some(ConnectionDef::Reference(s3_conn_ref)) => { - S3Connection::get(s3_conn_ref.as_ref(), client, namespace) - .await - .map_or(None, |s3c| s3c.spec.secret_class) - } - _ => None, + Some(ConnectionDef::Reference(res_name)) => Ok(S3BucketSpec { + connection: Some(ConnectionDef::Inline( + S3ConnectionSpec::get(res_name, client, namespace).await?, + )), + bucket_name: self.bucket_name.clone(), + }), + _ => Ok(self.clone()), } } } @@ -67,10 +71,20 @@ pub enum S3BucketDef { } impl S3BucketDef { - pub async fn secret_class(&self, client: &Client, namespace: Option) -> Option { + /// Returns an s3 bucket spec with an inlined connection. + pub async fn resolve( + &self, + client: &Client, + namespace: Option, + ) -> OperatorResult { match self { - S3BucketDef::Inline(s3_bucket) => s3_bucket.secret_class(client, namespace).await, - S3BucketDef::Reference(_s3_bucket) => todo!("get secret_class from bucket ref"), + S3BucketDef::Inline(s3_bucket) => s3_bucket.inlined(client, namespace).await, + S3BucketDef::Reference(_s3_bucket) => { + S3BucketSpec::get(_s3_bucket.as_str(), client, namespace.clone()) + .await? + .inlined(client, namespace) + .await + } } } } @@ -106,15 +120,16 @@ pub struct S3ConnectionSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub tls: Option, } -impl S3Connection { +impl S3ConnectionSpec { pub async fn get( resource_name: &str, client: &Client, namespace: Option, - ) -> OperatorResult { + ) -> OperatorResult { client - .get::(resource_name, namespace.as_deref()) + .get::(resource_name, namespace.as_deref()) .await + .map(|conn| conn.spec) .map_err(|_source| error::Error::MissingS3Connection { name: resource_name.to_string(), }) From b6d164a729a16b1340075bf8295c8e7baa02ff9d Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 27 Apr 2022 18:34:17 +0200 Subject: [PATCH 05/12] Added InlinedBucketSpec. --- src/commons/s3.rs | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/src/commons/s3.rs b/src/commons/s3.rs index baeea097d..7becd868e 100644 --- a/src/commons/s3.rs +++ b/src/commons/s3.rs @@ -50,19 +50,46 @@ impl S3BucketSpec { &self, client: &Client, namespace: Option, - ) -> OperatorResult { + ) -> OperatorResult { match self.connection.as_ref() { - Some(ConnectionDef::Reference(res_name)) => Ok(S3BucketSpec { - connection: Some(ConnectionDef::Inline( - S3ConnectionSpec::get(res_name, client, namespace).await?, - )), + Some(ConnectionDef::Reference(res_name)) => Ok(InlinedS3BucketSpec { + connection: Some(S3ConnectionSpec::get(res_name, client, namespace).await?,), + bucket_name: self.bucket_name.clone(), + }), + Some(ConnectionDef::Inline(conn_spec)) => Ok(InlinedS3BucketSpec { + bucket_name: self.bucket_name.clone(), + connection: Some(conn_spec.clone()), + }), + None => Ok(InlinedS3BucketSpec { bucket_name: self.bucket_name.clone(), + connection: None, }), - _ => Ok(self.clone()), } } } +pub struct InlinedS3BucketSpec { + pub bucket_name: Option, + pub connection: Option, +} + +impl InlinedS3BucketSpec { + pub fn host(&self) -> Option { + match self.connection.as_ref() { + Some(conn_spec) => conn_spec.host.clone(), + _ => None, + } + } + + pub fn secret_class(&self) -> Option { + match self.connection.as_ref() { + Some(conn_spec) => conn_spec.secret_class.clone(), + _ => None, + } + + } +} + #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub enum S3BucketDef { @@ -76,7 +103,7 @@ impl S3BucketDef { &self, client: &Client, namespace: Option, - ) -> OperatorResult { + ) -> OperatorResult { match self { S3BucketDef::Inline(s3_bucket) => s3_bucket.inlined(client, namespace).await, S3BucketDef::Reference(_s3_bucket) => { From 2b5fd2e2a7c740d8d55a9facd9fdd2d84148cb5b Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 27 Apr 2022 18:34:50 +0200 Subject: [PATCH 06/12] cargo fmt --- src/commons/s3.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/commons/s3.rs b/src/commons/s3.rs index 7becd868e..2055619cf 100644 --- a/src/commons/s3.rs +++ b/src/commons/s3.rs @@ -53,7 +53,7 @@ impl S3BucketSpec { ) -> OperatorResult { match self.connection.as_ref() { Some(ConnectionDef::Reference(res_name)) => Ok(InlinedS3BucketSpec { - connection: Some(S3ConnectionSpec::get(res_name, client, namespace).await?,), + connection: Some(S3ConnectionSpec::get(res_name, client, namespace).await?), bucket_name: self.bucket_name.clone(), }), Some(ConnectionDef::Inline(conn_spec)) => Ok(InlinedS3BucketSpec { @@ -86,7 +86,6 @@ impl InlinedS3BucketSpec { Some(conn_spec) => conn_spec.secret_class.clone(), _ => None, } - } } From f164c2f9d47c14c5dae9229f5ad92207b9a9c82e Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 28 Apr 2022 14:12:35 +0200 Subject: [PATCH 07/12] Formatting and docs. --- src/commons/s3.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/commons/s3.rs b/src/commons/s3.rs index 2055619cf..e8a86bb5a 100644 --- a/src/commons/s3.rs +++ b/src/commons/s3.rs @@ -1,7 +1,8 @@ //! Implementation of the bucket definition as described in -//! https://github.com/stackabletech/documentation/pull/177 +//! the //! -//! Operator CRDs are expected to use the [`S3BucketDef`] as an entry point to this module, +//! Operator CRDs are expected to use the [S3BucketDef] as an entry point to this module +//! and obtain an [InlinedS3BucketSpec] by calling [`S3BucketDef::resolve`]. //! use crate::commons::tls::Tls; use crate::error; @@ -10,6 +11,8 @@ use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +/// S3 bucket specification containing only the bucket name and an inlined or +/// referenced connection specification. #[derive(Clone, CustomResource, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] #[kube( group = "s3.stackable.tech", @@ -32,6 +35,7 @@ pub struct S3BucketSpec { } impl S3BucketSpec { + /// Convenience function to retrieve the spec of a S3 bucket resource from the K8S API service. pub async fn get( resource_name: &str, client: &Client, @@ -46,6 +50,7 @@ impl S3BucketSpec { }) } + /// Map &self to an [InlinedS3BucketSpec] by obtaining connection spec from the K8S API service if necessary pub async fn inlined( &self, client: &Client, @@ -68,12 +73,14 @@ impl S3BucketSpec { } } +/// Convenience struct with the connection spec inlined. pub struct InlinedS3BucketSpec { pub bucket_name: Option, pub connection: Option, } impl InlinedS3BucketSpec { + /// Shortcut to [S3ConnectionSpec::host] pub fn host(&self) -> Option { match self.connection.as_ref() { Some(conn_spec) => conn_spec.host.clone(), @@ -81,6 +88,7 @@ impl InlinedS3BucketSpec { } } + /// Shortcut to [S3ConnectionSpec::secret_class] pub fn secret_class(&self) -> Option { match self.connection.as_ref() { Some(conn_spec) => conn_spec.secret_class.clone(), @@ -89,6 +97,7 @@ impl InlinedS3BucketSpec { } } +/// Operators are expected to define fields for this type in order to work with S3 buckets. #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub enum S3BucketDef { @@ -97,7 +106,7 @@ pub enum S3BucketDef { } impl S3BucketDef { - /// Returns an s3 bucket spec with an inlined connection. + /// Returns an [InlinedS3BucketSpec]. pub async fn resolve( &self, client: &Client, @@ -115,6 +124,7 @@ impl S3BucketDef { } } +/// S3 connection definition used by [S3BucketSpec] #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub enum ConnectionDef { @@ -122,6 +132,7 @@ pub enum ConnectionDef { Reference(String), } +/// S3 connection definition as CRD. #[derive(CustomResource, Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] #[kube( group = "s3.stackable.tech", @@ -147,6 +158,7 @@ pub struct S3ConnectionSpec { pub tls: Option, } impl S3ConnectionSpec { + /// Convenience function to retrieve the spec of a S3 connection resource from the K8S API service. pub async fn get( resource_name: &str, client: &Client, From 31fd100aeaf9ba55d633ae713aea6a8b4b0c4c95 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 28 Apr 2022 14:26:21 +0200 Subject: [PATCH 08/12] Update CHANGELOG --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d8576576..e09dbe00f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- New commons::s3 module with common S3 connection structs ([#377]) + +[#377]: https://github.com/stackabletech/operator-rs/issues/377 + ## [0.17.0] - 2022-04-14 ### Changed From 392594dd11a1f75d811e7edbbd8b771b6443b95d Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 28 Apr 2022 15:39:25 +0200 Subject: [PATCH 09/12] Expose endpoint instead of host. --- src/commons/s3.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/commons/s3.rs b/src/commons/s3.rs index e8a86bb5a..c64e8e902 100644 --- a/src/commons/s3.rs +++ b/src/commons/s3.rs @@ -80,10 +80,13 @@ pub struct InlinedS3BucketSpec { } impl InlinedS3BucketSpec { - /// Shortcut to [S3ConnectionSpec::host] - pub fn host(&self) -> Option { + /// Build the endpoint URL from [S3Connection::host] and [S3Connection::port]. + pub fn endpoint(&self) -> Option { match self.connection.as_ref() { - Some(conn_spec) => conn_spec.host.clone(), + Some(conn_spec) => conn_spec.host.as_ref().map(|h| match conn_spec.port { + Some(p) => format!("s3a://{h}:{p}"), + _ => format!("s3a://{h}"), + }), _ => None, } } From 0ea25fe7f4a96511baba95427c1c9050a6d45272 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 28 Apr 2022 15:49:29 +0200 Subject: [PATCH 10/12] Update src/commons/s3.rs Co-authored-by: Malte Sander --- src/commons/s3.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commons/s3.rs b/src/commons/s3.rs index c64e8e902..e85f7014c 100644 --- a/src/commons/s3.rs +++ b/src/commons/s3.rs @@ -179,14 +179,14 @@ impl S3ConnectionSpec { #[cfg(test)] mod test { - use crate::commons::s3::ConnectionDef::Inline; + use crate::commons::s3::ConnectionDef; use crate::commons::s3::{S3BucketSpec, S3ConnectionSpec}; #[test] fn test_ser_inline() { let bucket = S3BucketSpec { bucket_name: Some("test-bucket-name".to_owned()), - connection: Some(Inline(S3ConnectionSpec { + connection: Some(ConnectionDef::Inline(S3ConnectionSpec { host: Some("host".to_owned()), port: Some(8080), secret_class: None, From 684054c02b51ac817cbce6c573f59b5aea0d922b Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 28 Apr 2022 16:02:08 +0200 Subject: [PATCH 11/12] Implement review feedback. --- src/commons/s3.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/commons/s3.rs b/src/commons/s3.rs index e85f7014c..c00ff32ae 100644 --- a/src/commons/s3.rs +++ b/src/commons/s3.rs @@ -39,10 +39,10 @@ impl S3BucketSpec { pub async fn get( resource_name: &str, client: &Client, - namespace: Option, + namespace: Option<&str>, ) -> OperatorResult { client - .get::(resource_name, namespace.as_deref()) + .get::(resource_name, namespace) .await .map(|crd| crd.spec) .map_err(|_source| error::Error::MissingS3Bucket { @@ -54,7 +54,7 @@ impl S3BucketSpec { pub async fn inlined( &self, client: &Client, - namespace: Option, + namespace: Option<&str>, ) -> OperatorResult { match self.connection.as_ref() { Some(ConnectionDef::Reference(res_name)) => Ok(InlinedS3BucketSpec { @@ -113,12 +113,12 @@ impl S3BucketDef { pub async fn resolve( &self, client: &Client, - namespace: Option, + namespace: Option<&str>, ) -> OperatorResult { match self { S3BucketDef::Inline(s3_bucket) => s3_bucket.inlined(client, namespace).await, S3BucketDef::Reference(_s3_bucket) => { - S3BucketSpec::get(_s3_bucket.as_str(), client, namespace.clone()) + S3BucketSpec::get(_s3_bucket.as_str(), client, namespace) .await? .inlined(client, namespace) .await @@ -165,10 +165,10 @@ impl S3ConnectionSpec { pub async fn get( resource_name: &str, client: &Client, - namespace: Option, + namespace: Option<&str>, ) -> OperatorResult { client - .get::(resource_name, namespace.as_deref()) + .get::(resource_name, namespace) .await .map(|conn| conn.spec) .map_err(|_source| error::Error::MissingS3Connection { From ede8367d30ab50ce00f93928094e76d194b0afc9 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 28 Apr 2022 16:07:22 +0200 Subject: [PATCH 12/12] Fix doc errors. --- src/commons/s3.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commons/s3.rs b/src/commons/s3.rs index c00ff32ae..1f7f1b01d 100644 --- a/src/commons/s3.rs +++ b/src/commons/s3.rs @@ -80,7 +80,7 @@ pub struct InlinedS3BucketSpec { } impl InlinedS3BucketSpec { - /// Build the endpoint URL from [S3Connection::host] and [S3Connection::port]. + /// Build the endpoint URL from [S3ConnectionSpec::host] and [S3ConnectionSpec::port]. pub fn endpoint(&self) -> Option { match self.connection.as_ref() { Some(conn_spec) => conn_spec.host.as_ref().map(|h| match conn_spec.port {