diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..aaef0ac --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/target +.opencode diff --git a/README.md b/README.md new file mode 100644 index 0000000..d51daed --- /dev/null +++ b/README.md @@ -0,0 +1,163 @@ +# K8s secret generator + +> *AI*: This was also an exploration of opencode features, so there is some amount of AI-generated boilerplate. + +## Features + +### Generate namespaced secrets from specification + +To generate arbitrary secret deploy `RandomSecret` resource: + +```yaml +apiVersion: generator.kuyaseg.spushk.ru/v1alpha1 +kind: RandomSecret +metadata: + name: secret + namespace: default +spec: + # Specify all keys to generate and put into secret + keys: + # Binary key - random byte sequence + - kind: binary + # The key to put generated value to + key: binary-key + #If changes after last reconcilation, will force regeneration with new value + version: 0 + # Set size of value in bytes + size: 64 + # Password key - guaranteed UTF-8 character sequence with repeat period >= 3 + - kind: password + # The key to put generated value to + key: password-key + #If changes after last reconcilation, will force regeneration with new value + version: 0 + # Set length of password in symbols + length: 24 + # Specify which symbols are allowed to use when generating password + dictionary: "abcdefghijklmnopqrstuvwxyz0123456789_" + # Adjust data, stringData and metadata of generated Secret + template: {} +``` + +#### SSH keys + +To generate SSH key deploy `RandomSsh` resource: + +```yaml +apiVersion: generator.kuyaseg.spushk.ru/v1alpha1 +kind: RandomSsh +metadata: + name: ssh-secret + namespace: default +spec: + # Configure keypair configuration + keypair: + # If changes after last reconcilation, will force regeneration with new keypair + version: 0 + # `ed25519` is default. `rsa` is WIP + algorithm: ed25519 + # optional field to set pubkey comment + comment: mysecret + # Configure SSH private key + identity: + # The key to put generated identity to + key: private-key + # Adjust data, stringData and metadata of generated Secret + template: {} + # Configure SSH public key + pubkey: + # The key to put generated identity to + key: public-key + # Adjust data, stringData and metadata of generated Secret + template: {} +``` + +The following template properties have additional validation: +- `metadata.owner_references`: controller owner is not allowed + +When reconciled, this resource will produce two `v1/Secret` resources in the same namespace: +- `-identity`: private SSH key, which is stored in specified key +- `-pubkey`: public SSH key which is stored in specified key + +Both resources inherit namespace and name (as prefix) from `RandomSsh` resource and are immutable. + +### Deterministic secrets + +> ⚠️ This feature exists primarily for testing purposes, where security is less of a concern then repeatability. +> +> While there are use-cases for predictable secrets in production, you should consider [ESO](https://external-secrets.io) +> with secure external storage to share secrets between clusters and keep secrets stable. +> +> If the seed secret gets compromised - every past and future generated secret can be easliy computed. + +Kuyaseg is able to provide deterministic source for secret and SSH keypair generation, offering idempotent sequence of secret values. + +With deterministic secrets value remains the same even if specification was deleted and created anew. + +This could be useful for testing, debugging and learning in local environments. + +Deterministic secrets are created with `DeterministicSecret` and `DeterministicSsh` resources, matching `RandomSecret` and `RandomSsh` resource schema: + +```yaml +# Schema matches the Secret from generator.kuyaseg.spushk.ru/v1alpha1 +apiVersion: generator.kuyaseg.spushk.ru/v1alpha1 +kind: DeterministicSecret +metadata: + name: deterministic-secret + namespace: testing +spec: + keys: + - key: debug-password + kind: password + length: 12 + template: + stringData: + username: debug-user +``` + +Secret values generated by deterministic generator depend on: +- Global `seed` value set for controller +- `metadata.name` and `metadata.namespace` of `DeterministicSecret` resource +- `spec.keys.*.key` and `spec.key.*.version` for each generated key specified in `DeterministicSecret` resource + +Thus, if seed secret remains stable and DeterministicSecret specification does not change - generated keys remain stable and predictable + +## Feature comparison + +We know there are plenty of decent, mature and well-adopted solutions like the one by [mittwald](https://github.com/mittwald/kubernetes-secret-generator) and even [ESO](https://external-secrets.io) which can also generate secrets. + +Here is a comparison. + +||Kuyaseg|kubernetes-secret-generator|ESO|| +|-|-|-|-|-| +|Generate cryptographically secure namespaced secrets|Hc128 CSPRNG, allows to specify generated **keys per resource**|Cryptographically secure, allows to specify generated **keys per resource**|Cryptographically secure, but generated keys are specified **per generator**|| +|Generate SSH keypairs|Ed25519 and RSA OpenSSH keypairs, private and public keys **stored in separate secrets**|Both secret and public keys are **stored in one secret**|Not supported|| +|Deterministic, reproducible secret generation|Available, optional, based on provided secret data (e.g. SealedSecrets). Stable across clusters|Not supported|Not supported|| +|Secret value stability|Created secrets are |Stable across reconcilations, but **changes on secret re-creation**|Only with real **external secret storage** (which ESO is designed for)|| +|Target secret template|Respects almost full `Secret` (v1) spec, including |[Not supported](https://github.com/mittwald/kubernetes-secret-generator/issues/88)|Allows to specity template in `ExternalSecret`|| + +## Security + +⚠️ This software have never received independent security audit. I am not a security expert. + +--- + +For security issues, please, view the [security policy](https://source.spushk.ru/security) and kindly e-mail +us at [reports@spushk.ru](mailto:reports@spushk.ru?subject=%5BSECURITY%5D%3A%20describe%20the%20issue&body=Thank%20you%20for%20submitting%20security%20report%20%E2%9D%A4%EF%B8%8F%0AWe%20kindly%20ask%20you%20not%20to%20publically%20disclose%20any%20information%20related%20to%20your%20findings%20just%20yet.%0APlease%2C%20fill%20out%20the%20following%20sections%20and%20provide%20any%20additional%20information%20you%27re%20willing%20to%20share.%0A%0A-------------------------------------------------------------------------------%20FINDING%0AWhat%20is%20the%20substance%20of%20an%20issue%3F%0A%0A-------------------------------------------------------------------------------%20AFFECTED%20SOFTWARE%0AWhich%20product%20and%20which%20versions%20is%20affected%3F%0A%0A-------------------------------------------------------------------------------%20REPRODUCTION%0APlease%2C%20share%20PoC%20if%20you%20have%20one%0A%0A-------------------------------------------------------------------------------%20DISCLOSURE%0AWhat%20are%20you%20expectations%20on%20disclosure%3F%0AWe%20kindly%20ask%20you%20not%20to%20share%20any%20information%20publically%0Aat%20least%20for%2090%20days%20since%20this%20report.%0A%0A-------------------------------------------------------------------------------%20CONTACT%0AHow%20do%20we%20contact%20you%3F) + +--- + +### Secret generation + +To produce random secrets (`RandomSecret`, `RandomSshSecret`), operator uses [aws-lc-rs+fips](https://docs.rs/aws-lc-rs/latest/aws_lc_rs/). + +To produce deterministic secrets (`DeterministicSecret`, `DeterministicSsh`), operator uses [Hc128](https://docs.rs/rand_hc/latest/rand_hc/) CSPRNG. + +### Secret persistence & memory + +Random secrets defined by `RandomSecret` and `RandomSsh` are unstable. Deletion of generated `Secret` will lead to new random value on next reconcilation. + +Deterministic secrets defined by `DeterministicSecret` and `DeterministicSsh` are stable as long as the base `seed` remains unchanged. +Deletion of generated `Secret` resources will lead to the same pseudo-random value put into the secret on next reconcilation as long as spec remains the same. + +Best effort is taken to ensure that secrets do not remain in memory longer then needed and/or after crush. diff --git a/kuyaseg-controller/src/generator.rs b/kuyaseg-controller/src/generator.rs index f56fb60..00baf8f 100644 --- a/kuyaseg-controller/src/generator.rs +++ b/kuyaseg-controller/src/generator.rs @@ -37,6 +37,7 @@ pub fn str_bytes( ) -> Result, GeneratorErr> { let mut buf = String::with_capacity(length); + // TODO: unique may leak dictionary..? let dictionary = Zeroizing::new(dictionary.unique().collect::>()); let distribution = rand::distr::slice::Choose::new(&dictionary).map_err(|_| GeneratorErr::EmptyDistribution)?; @@ -69,12 +70,12 @@ pub fn str_bytes( } if buf.len() != length { - Err(GeneratorErr::RngStuck { + return Err(GeneratorErr::RngStuck { reason: format!("Unable to sample {length} characters"), - }) - } else { - Ok(buf.into_bytes()) + }); } + + Ok(buf.into_bytes()) } /// Generate SSH keypair, encoded for OpenSSH diff --git a/kuyaseg-controller/src/reporting/k8s_event.rs b/kuyaseg-controller/src/reporting/k8s_event.rs index 5bd1294..80ce3b8 100644 --- a/kuyaseg-controller/src/reporting/k8s_event.rs +++ b/kuyaseg-controller/src/reporting/k8s_event.rs @@ -25,7 +25,7 @@ impl K8sEventRecorder { .recorder .publish(&event.event, &event.object) .await - .inspect_err(|e| tracing::error!(no_event=true, error=?e, "Failed to record K8s event")); + .inspect_err(|e| tracing::error!(k8s_no_event=true, error=?e, "Failed to record K8s event")); } } diff --git a/kuyaseg-controller/src/reporting/mod.rs b/kuyaseg-controller/src/reporting/mod.rs index 240a9c4..d24cbfc 100644 --- a/kuyaseg-controller/src/reporting/mod.rs +++ b/kuyaseg-controller/src/reporting/mod.rs @@ -126,7 +126,7 @@ fn init_k8s_events(client: kube::Client, joinset: &mut JoinSet<()>, ct: Cancella } queue.push_back(ev); if queue.len() > WARN_THRESHOLD && !warned { - tracing::warn!(no_event=true, queue_len=queue.len(), "K8s event queue exceeded threshold"); + tracing::warn!(k8s_no_event=true, queue_len=queue.len(), "K8s event queue exceeded threshold"); warned = true; } else if queue.len() <= WARN_THRESHOLD { warned = false; @@ -135,7 +135,7 @@ fn init_k8s_events(client: kube::Client, joinset: &mut JoinSet<()>, ct: Cancella rec_limit = 4; } Err(e) => { - tracing::error!(no_event=true, debug=?e, "Failed to receive k8s event from tracing"); + tracing::error!(k8s_no_event=true, debug=?e, "Failed to receive k8s event from tracing"); rx = rtx.subscribe(); // dumb recover strategy tokio::time::sleep(Duration::from_secs(3)).await; rec_limit -= 1; @@ -180,7 +180,7 @@ fn init_pro_metrics(joinset: &mut JoinSet<()>, ct: CancellationToken) -> Sender< rec_limit = 4; } Err(e) => { - tracing::error!(no_pro=true, debug=?e, "Failed to receive prometheus measurement from tracing"); + tracing::error!(k8s_no_pro=true, debug=?e, "Failed to receive prometheus measurement from tracing"); rx = rtx.subscribe(); // dumb recover strategy; tokio::time::sleep(Duration::from_secs(3)).await; rec_limit -= 1; @@ -205,11 +205,11 @@ fn init_pro_metrics(joinset: &mut JoinSet<()>, ct: CancellationToken) -> Sender< metrics.register(&mut registry); }, _ = registration_cancellation.cancelled() => { - tracing::warn!(no_pro=true, "Stopped publishing prometheus metrics: consumer job cancelled"); + tracing::warn!(k8s_no_pro=true, "Stopped publishing prometheus metrics: consumer job cancelled"); break; } _ = ctk.cancelled() => { - tracing::warn!(no_pro=true, "Stopped publishing prometheus metrics: shutdown"); + tracing::warn!(k8s_no_pro=true, "Stopped publishing prometheus metrics: shutdown"); break; } } @@ -251,7 +251,7 @@ fn init_otel(joinset: &mut JoinSet<()>, ct: CancellationToken) -> anyhow::Result } Err(e) => { tracing::error!( - no_otel = true, + k8s_no_otel = true, debug = ?e, "Failed to receive otel observation from tracing" ); @@ -263,7 +263,7 @@ fn init_otel(joinset: &mut JoinSet<()>, ct: CancellationToken) -> anyhow::Result } } } - tracing::error!(no_otel = true, "OpenTelemetry reporting from tracing stopped"); + tracing::error!(k8s_no_otel = true, "OpenTelemetry reporting from tracing stopped"); }); Ok(tx) diff --git a/kuyaseg-controller/src/reporting/prometheus.rs b/kuyaseg-controller/src/reporting/prometheus.rs index 4d0d7ba..c648ffb 100644 --- a/kuyaseg-controller/src/reporting/prometheus.rs +++ b/kuyaseg-controller/src/reporting/prometheus.rs @@ -47,6 +47,7 @@ impl ProMetrics { } /// Start measuring reconcile duration. Registers on `TimedMeasurer` drop + // TODO: use this pub fn measure_reconcilation(&self) -> TimedMeasurer { self.reconcile_metrics.measure() } diff --git a/kuyaseg-controller/src/reporting/tracing_evt.rs b/kuyaseg-controller/src/reporting/tracing_evt.rs index 0e1c8fa..7663f3a 100644 --- a/kuyaseg-controller/src/reporting/tracing_evt.rs +++ b/kuyaseg-controller/src/reporting/tracing_evt.rs @@ -302,9 +302,9 @@ impl Visit for SpanFields { fn record_bool(&mut self, field: &tracing::field::Field, value: bool) { match field.name() { - "no_event" => self.no_event = Some(value), - "no_pro" => self.no_pro = Some(value), - "no_otel" => self.no_otel = Some(value), + "k8s_no_event" => self.no_event = Some(value), + "k8s_no_pro" => self.no_pro = Some(value), + "k8s_no_otel" => self.no_otel = Some(value), _ => {} } } @@ -361,9 +361,9 @@ impl Visit for EventFields { fn record_bool(&mut self, field: &tracing::field::Field, value: bool) { match field.name() { "critical" => self.critical = Some(value), - "no_event" => self.no_event = Some(value), - "no_pro" => self.no_pro = Some(value), - "no_otel" => self.no_otel = Some(value), + "k8s_no_event" => self.no_event = Some(value), + "k8s_no_pro" => self.no_pro = Some(value), + "k8s_no_otel" => self.no_otel = Some(value), _ => {} } } diff --git a/kuyaseg-controller/src/resource/generated/secret/mod.rs b/kuyaseg-controller/src/resource/generated/secret/mod.rs index cfb12e3..b919aba 100644 --- a/kuyaseg-controller/src/resource/generated/secret/mod.rs +++ b/kuyaseg-controller/src/resource/generated/secret/mod.rs @@ -52,12 +52,7 @@ impl GeneratedSecretStatus { return false; }; - (stored.0 == spec.version) - && aws_lc_rs::constant_time::verify_slices_are_equal( - stored.1.as_bytes(), - spec.shape.stable_hash().as_bytes(), - ) - .is_ok() + (stored.0 == spec.version) && (stored.1 == spec.shape.stable_hash()) } } @@ -68,7 +63,13 @@ impl GeneratedSecretSpec { /// This method family assumes that upstream resource has `GeneratedSecretStatus` status. /// /// I wrote this because - hell no - I won't write this code twice for both Random and Deterministic secrets! - async fn reconcile_spec(&self, name: &str, namespace: &str, client: kube::Client) -> anyhow::Result<()> + async fn reconcile_spec( + &self, + name: &str, + namespace: &str, + client: kube::Client, + uid: Option<&str>, + ) -> anyhow::Result<()> where T: HasSecretGenerator + Resource @@ -103,20 +104,28 @@ impl GeneratedSecretSpec { .await .context("Update existing Secret")?, None => self - .create_new_from_spec::(name, namespace, client) + .create_new_from_spec::(name, namespace, client, uid) .await - .context("Update existing Secret")?, + .context("Create new Secret")?, } Ok(()) } - async fn create_new_from_spec(&self, name: &str, namespace: &str, client: kube::Client) -> anyhow::Result<()> + async fn create_new_from_spec( + &self, + name: &str, + namespace: &str, + client: kube::Client, + uid: Option<&str>, + ) -> anyhow::Result<()> where T: HasSecretGenerator + Resource + DeserializeOwned, { + use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference; + let self_api = Api::::namespaced(client.clone(), namespace); let target_api = Api::::namespaced(client.clone(), namespace); @@ -127,7 +136,19 @@ impl GeneratedSecretSpec { .collect::>(); // first we try with dry-run, no need to guard as no data here - let partial_target = self.template.to_safe_partial_secret(Some(false), Some("Opaque".into())); + let owner_ref = uid.map(|uid| OwnerReference { + api_version: T::api_version(&()).to_string(), + kind: T::kind(&()).to_string(), + name: name.to_string(), + uid: uid.to_string(), + controller: Some(true), + ..Default::default() + }); + let partial_target = { + let mut secret = self.template.to_safe_partial_secret(Some(false), Some("Opaque".into())); + secret.metadata.owner_references = owner_ref.as_ref().map(|oref| vec![oref.clone()]); + secret + }; let patch = kube::api::Patch::Apply(json!({ "apiVersion": T::api_version(&()), "kind": T::kind(&()), @@ -201,8 +222,7 @@ impl GeneratedSecretSpec { }) .context("Create Secret")?; - // TODO: delete created secret if this fails or not..? - self_api + if let Err(e) = self_api .patch_status( name, &kube::api::PatchParams { @@ -213,8 +233,18 @@ impl GeneratedSecretSpec { }, &patch, ) - .await.inspect_err(|e| tracing::error!(debug=?e, reason="SelfUpdateFailed", action = "CreateNew", "Failed to update self status")) - .context("Update status")?; + .await + .inspect_err(|e| tracing::error!(debug=?e, reason="SelfUpdateFailed", action = "CreateNew", "Failed to update self status")) + .context("Update status") + { + tracing::error!( + reason = "SecretOrphanCleanup", + action = "CreateNew", + "Deleting orphan secret after failed status update" + ); + target_api.delete(name, &kube::api::DeleteParams::default()).await.context("Orphan secret on fail")?; + return Err(e); + } } Ok(()) @@ -344,9 +374,11 @@ impl GeneratedSecretShape { let mut hasher = blake3::Hasher::new(); match self { Self::Bytes { size } => { + hasher.update(b"bytes"); hasher.update(&size.to_le_bytes()); } Self::Password { length, dictionary } => { + hasher.update(b"password"); hasher.update(&length.to_le_bytes()); hasher.update(dictionary.as_bytes()); } diff --git a/kuyaseg-controller/src/resource/generated/secret/v1alpha1.rs b/kuyaseg-controller/src/resource/generated/secret/v1alpha1.rs index 5df999b..1111bf6 100644 --- a/kuyaseg-controller/src/resource/generated/secret/v1alpha1.rs +++ b/kuyaseg-controller/src/resource/generated/secret/v1alpha1.rs @@ -58,8 +58,14 @@ impl RandomSecret { let kind = Self::kind(&()); let api = Self::api_version(&()); let name = &self.name_any(); - let _span_nons = - tracing::info_span!("RandomSecret.reconcile", kind = kind.as_ref(), api = api.as_ref(), name).entered(); + let _span_nons = tracing::info_span!( + "RandomSecret.reconcile", + k8s_kind = kind.as_ref(), + k8s_api = api.as_ref(), + k8s_name = name, + k8s_uid = self.uid(), + ) + .entered(); let Some(namespace) = self.namespace() else { tracing::error!( @@ -69,10 +75,11 @@ impl RandomSecret { ); return Err(anyhow::Error::msg("RandomSecret needed namespace")); }; - let _span = tracing::info_span!("RandomSecret.reconcile.ns", ns = namespace).entered(); + let _span = tracing::info_span!("RandomSecret.reconcile.ns", k8s_ns = namespace).entered(); - self.spec.0.reconcile_spec::(name, &namespace, client).await.inspect_err(|e| tracing::error!(debug=?e, reason="RandomSecretReconcilationFailed", action = "Reconcile", "Failed to reconcile RandomSecret {namespace}{name}")).context("Reconcile")?; + self.spec.0.reconcile_spec::(name, &namespace, client, self.uid().as_deref()).await.inspect_err(|e| tracing::error!(debug=?e, reason="RandomSecretReconciliationFailed", action = "Reconcile", "Failed to reconcile RandomSecret {namespace}{name}")).context("Reconcile")?; + // TODO: verify if all desired events fire Ok(Action::await_change()) } } @@ -130,9 +137,10 @@ impl DeterministicSecret { let name = &self.name_any(); let _span_nons = tracing::info_span!( "DeterministicSecret.reconcile", - kind = kind.as_ref(), - api = api.as_ref(), - name + k8s_kind = kind.as_ref(), + k8s_api = api.as_ref(), + k8s_name = name, + k8s_uid = self.uid(), ) .entered(); @@ -144,17 +152,18 @@ impl DeterministicSecret { ); return Err(anyhow::Error::msg("DeterministicSecret needed namespace")); }; - let _span = tracing::info_span!("DeterministicSecret.reconcile.ns", ns = namespace).entered(); + let _span = tracing::info_span!("DeterministicSecret.reconcile.ns", k8s_ns = namespace).entered(); - self.spec.0.reconcile_spec::(name, &namespace, client).await.inspect_err(|e| tracing::error!(debug=?e, reason="DeterministicSecretReconcilationFailed", action = "Reconcile", "Failed to reconcile DeterministicSecret {namespace}{name}")).context("Reconcile")?; + self.spec.0.reconcile_spec::(name, &namespace, client, self.uid().as_deref()).await.inspect_err(|e| tracing::error!(debug=?e, reason="DeterministicSecretReconciliationFailed", action = "Reconcile", "Failed to reconcile DeterministicSecret {namespace}{name}")).context("Reconcile")?; + // TODO: verify if all desired events fire Ok(Action::await_change()) } } impl HasSecretGenerator for DeterministicSecret { fn generate_data(spec: &GeneratedSecretSpec) -> anyhow::Result> { - let seed = std::env::var("KUYASEG_SEED").inspect_err(|e| tracing::error!(debug=?e, reason="DeterministicSecretNeedSeed", "Refuse to generate deterministic secret without seet. Set with KUYASEG_SEED env")).context("Get external seed")?; + let seed = std::env::var("KUYASEG_SEED").inspect_err(|e| tracing::error!(debug=?e, reason="DeterministicSecretNeedSeed", "Refuse to generate deterministic secret without seed. Set with KUYASEG_SEED env")).context("Get external seed")?; let mut data = BTreeMap::new(); for key in spec.keys.iter() { diff --git a/opencode.jsonc b/opencode.jsonc new file mode 100644 index 0000000..ef7fc30 --- /dev/null +++ b/opencode.jsonc @@ -0,0 +1,13 @@ +{ + "$schema": "https://opencode.ai/config.json", + "watcher": { + "ignore": [".opencode/node_modules", ".opencode/package.json", ".opencode/package-lock.json"], + }, + "permission": { + "bash": "ask", + "read": { + ".env": "ask", + ".envrc": "ask", + }, + }, +}