kubeworks/kube-yaseg
Watch
1
0
Fork
You've already forked kube-yaseg
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller/reporting): rate limit k8s events & add external cancellation

Browse source
This commit is contained in:
spushk 2026-05-16 04:49:34 +05:00 • committed by spushk
commit a42cc2e7ac
Signed by: spushk
GPG key ID: 7797B761E105E8AA
3 changed files with 124 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

161
kuyaseg-controller/src/reporting/mod.rs
View file

@ -1,3 +1,4 @@
use std::collections::VecDeque;
use std::sync::Arc; use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
@ -49,7 +50,7 @@ pub struct ReportingInit {
pub handle: JoinSet<()>, pub handle: JoinSet<()>,
} }
pub fn init(k8s_client: kube::Client) -> anyhow::Result<ReportingInit> { pub fn init(k8s_client: kube::Client, ct: CancellationToken) -> anyhow::Result<ReportingInit> {
let config = ReportingConfig::builder() let config = ReportingConfig::builder()
.env() .env()
.load() .load()
@ -59,9 +60,14 @@ pub fn init(k8s_client: kube::Client) -> anyhow::Result<ReportingInit> {
let k8s_tx = config let k8s_tx = config
.k8s_events .k8s_events
.then(|| init_k8s_events(k8s_client, &mut reporting_jobs)); .then(|| init_k8s_events(k8s_client, &mut reporting_jobs, ct.clone()));
let pro_tx = config.prometheus.then(|| init_pro_metrics(&mut reporting_jobs)); let pro_tx = config
let otel_tx = config.opentelemetry.then(|| init_otel(&mut reporting_jobs)); .prometheus
.then(|| init_pro_metrics(&mut reporting_jobs, ct.clone()));
let otel_tx = config
.opentelemetry
.then(|| init_otel(&mut reporting_jobs, ct.clone()))
.transpose()?;
Ok(ReportingInit { Ok(ReportingInit {
tracing_layer: TracingK8sEventLayer::new(TracingK8sEventLayerConfig { tracing_layer: TracingK8sEventLayer::new(TracingK8sEventLayerConfig {
@ -73,7 +79,7 @@ pub fn init(k8s_client: kube::Client) -> anyhow::Result<ReportingInit> {
}) })
} }
fn init_k8s_events(client: kube::Client, joinset: &mut JoinSet<()>) -> Sender<Arc<K8sEvent>> { fn init_k8s_events(client: kube::Client, joinset: &mut JoinSet<()>, ct: CancellationToken) -> Sender<Arc<K8sEvent>> {
use kube::runtime::events::{ use kube::runtime::events::{
Recorder, Recorder,
Reporter, Reporter,
@ -81,7 +87,10 @@ fn init_k8s_events(client: kube::Client, joinset: &mut JoinSet<()>) -> Sender<Ar
use crate::reporting::k8s_event::K8sEventRecorder; use crate::reporting::k8s_event::K8sEventRecorder;
let (tx, mut rx) = broadcast::channel::<Arc<K8sEvent>>(128); const WARN_THRESHOLD: usize = 10;
const MAX_CAPACITY: usize = 100;
let (tx, mut rx) = broadcast::channel::<Arc<K8sEvent>>(512);
let reporter = Reporter { let reporter = Reporter {
controller: env!("CARGO_PKG_NAME").into(), controller: env!("CARGO_PKG_NAME").into(),
instance: std::env::var("CONTROLLER_POD_NAME").ok(), instance: std::env::var("CONTROLLER_POD_NAME").ok(),
@ -91,36 +100,65 @@ fn init_k8s_events(client: kube::Client, joinset: &mut JoinSet<()>) -> Sender<Ar
let rtx = tx.clone(); // for dumb recovery let rtx = tx.clone(); // for dumb recovery
joinset.spawn(async move { joinset.spawn(async move {
let mut queue: VecDeque<Arc<K8sEvent>> = VecDeque::new();
let mut warned = false;
let mut drain_tick = tokio::time::interval(Duration::from_secs(6)); // 10 event/min
let mut rec_limit = 4; let mut rec_limit = 4;
while rec_limit > 0 { while rec_limit > 0 {
match rx.recv().await { tokio::select! {
Ok(ev) => { biased;
recorder.record(&ev).await; _ = ct.cancelled() => break,
rec_limit = 4; _ = drain_tick.tick() => {
} if let Some(ev) = queue.pop_front() {
Err(e) => { recorder.record(&ev).await;
tracing::error!(no_event=true, debug=?e, "Failed to receive k8s event from tracing"); }
rx = rtx.subscribe(); // dumb recover strategy },
tokio::time::sleep(Duration::from_secs(3)).await; msg = rx.recv() => {
rec_limit -= 1; match msg {
Ok(ev) => {
if queue.is_empty() {
recorder.record(&ev).await;
warned = false;
} else {
if queue.len() >= MAX_CAPACITY {
queue.pop_front();
}
queue.push_back(ev);
if queue.len() > WARN_THRESHOLD && !warned {
tracing::warn!(no_event=true, queue_len=queue.len(), "K8s event queue exceeded threshold");
warned = true;
} else if queue.len() <= WARN_THRESHOLD {
warned = false;
}
}
rec_limit = 4;
}
Err(e) => {
tracing::error!(no_event=true, debug=?e, "Failed to receive k8s event from tracing");
rx = rtx.subscribe(); // dumb recover strategy
tokio::time::sleep(Duration::from_secs(3)).await;
rec_limit -= 1;
}
}
} }
} }
} }
tracing::error!( tracing::error!(
no_event = true, no_event = true,
"K8s Event reporting from tracing stopped from limit reached" "K8s Event reporting from tracing stopped"
); );
}); });
tx tx
} }
fn init_pro_metrics(joinset: &mut JoinSet<()>) -> Sender<Arc<ProLabel>> { fn init_pro_metrics(joinset: &mut JoinSet<()>, ct: CancellationToken) -> Sender<Arc<ProLabel>> {
use prometheus_client::registry::Registry; use prometheus_client::registry::Registry;
use crate::reporting::prometheus::ProMetrics; use crate::reporting::prometheus::ProMetrics;
let (tx, mut rx) = broadcast::channel::<Arc<ProLabel>>(128); let (tx, mut rx) = broadcast::channel::<Arc<ProLabel>>(512);
let metrics = ProMetrics::new(); let metrics = ProMetrics::new();
let registration_cancellation = CancellationToken::new(); let registration_cancellation = CancellationToken::new();
@ -128,28 +166,37 @@ fn init_pro_metrics(joinset: &mut JoinSet<()>) -> Sender<Arc<ProLabel>> {
let rtx = tx.clone(); let rtx = tx.clone();
let ctk = registration_cancellation.clone(); let ctk = registration_cancellation.clone();
let promt = metrics.clone(); let promt = metrics.clone();
let consumer_ct = ct.clone();
joinset.spawn(async move { joinset.spawn(async move {
let mut rec_limit = 4; let mut rec_limit = 4;
while rec_limit > 0 { while rec_limit > 0 {
match rx.recv().await { tokio::select! {
Ok(met) => { biased;
promt.measure(&met); _ = consumer_ct.cancelled() => break,
} msg = rx.recv() => {
Err(e) => { match msg {
tracing::error!(no_pro=true, debug=?e, "Failed to receive prometheus measurement from tracing"); Ok(met) => {
rx = rtx.subscribe(); // dumb recover strategy; promt.measure(&met);
tokio::time::sleep(Duration::from_secs(3)).await; rec_limit = 4;
rec_limit -= 1; }
Err(e) => {
tracing::error!(no_pro=true, debug=?e, "Failed to receive prometheus measurement from tracing");
rx = rtx.subscribe(); // dumb recover strategy;
tokio::time::sleep(Duration::from_secs(3)).await;
rec_limit -= 1;
}
}
} }
} }
} }
ctk.cancel(); ctk.cancel();
tracing::error!( tracing::error!(
no_pro = true, no_pro = true,
"Prometheus measurement from tracing stopped from limit reached" "Prometheus measurement from tracing stopped"
); );
}); });
let ctk = ct.clone();
joinset.spawn(async move { joinset.spawn(async move {
let mut registry = Registry::default(); let mut registry = Registry::default();
loop { loop {
@ -158,7 +205,11 @@ fn init_pro_metrics(joinset: &mut JoinSet<()>) -> Sender<Arc<ProLabel>> {
metrics.register(&mut registry); metrics.register(&mut registry);
}, },
_ = registration_cancellation.cancelled() => { _ = registration_cancellation.cancelled() => {
tracing::warn!(no_pro=true, "Stopped publishing prometheus metrics: job cancelled"); tracing::warn!(no_pro=true, "Stopped publishing prometheus metrics: consumer job cancelled");
break;
}
_ = ctk.cancelled() => {
tracing::warn!(no_pro=true, "Stopped publishing prometheus metrics: shutdown");
break; break;
} }
} }
@ -168,48 +219,52 @@ fn init_pro_metrics(joinset: &mut JoinSet<()>) -> Sender<Arc<ProLabel>> {
tx tx
} }
fn init_otel(joinset: &mut JoinSet<()>) -> Sender<Arc<OtelObservation>> { fn init_otel(joinset: &mut JoinSet<()>, ct: CancellationToken) -> anyhow::Result<Sender<Arc<OtelObservation>>> {
use crate::reporting::otel::OtelAggregator; use crate::reporting::otel::OtelAggregator;
let exporter = opentelemetry_otlp::LogExporter::builder() let exporter = opentelemetry_otlp::LogExporter::builder()
.with_tonic() .with_tonic()
.build() .build()
.expect("Failed to create OTLP log exporter"); .context("Failed to build OTLP log exporter")?;
let provider = opentelemetry_sdk::logs::SdkLoggerProvider::builder() let provider = opentelemetry_sdk::logs::SdkLoggerProvider::builder()
.with_batch_exporter(exporter) .with_batch_exporter(exporter)
.build(); .build();
let aggregator = OtelAggregator::new(provider); let aggregator = Arc::new(OtelAggregator::new(provider));
let task_aggregator = aggregator.clone();
let (tx, mut rx) = broadcast::channel::<Arc<OtelObservation>>(128); let (tx, mut rx) = broadcast::channel::<Arc<OtelObservation>>(512);
let rtx = tx.clone(); let rtx = tx.clone();
joinset.spawn(async move { joinset.spawn(async move {
let mut rec_limit = 4; let mut rec_limit = 4;
while rec_limit > 0 { while rec_limit > 0 {
match rx.recv().await { tokio::select! {
Ok(obs) => { biased;
aggregator.observe(&obs); _ = ct.cancelled() => break,
rec_limit = 4; msg = rx.recv() => {
} match msg {
Err(e) => { Ok(obs) => {
tracing::error!( task_aggregator.observe(&obs);
no_otel = true, rec_limit = 4;
debug = ?e, }
"Failed to receive otel observation from tracing" Err(e) => {
); tracing::error!(
rx = rtx.subscribe(); no_otel = true,
tokio::time::sleep(Duration::from_secs(3)).await; debug = ?e,
rec_limit -= 1; "Failed to receive otel observation from tracing"
);
rx = rtx.subscribe();
tokio::time::sleep(Duration::from_secs(3)).await;
rec_limit -= 1;
}
}
} }
} }
} }
tracing::error!( tracing::error!(no_otel = true, "OpenTelemetry reporting from tracing stopped");
no_otel = true,
"OpenTelemetry reporting from tracing stopped from limit reached"
);
}); });
tx Ok(tx)
} }

6
kuyaseg-controller/src/reporting/otel.rs
View file

@ -98,3 +98,9 @@ impl OtelAggregator {
todo!("Metrics are not implemented and should not be used for now") todo!("Metrics are not implemented and should not be used for now")
} }
} }
impl Drop for OtelAggregator {
fn drop(&mut self) {
self.force_flush();
}
}

34
kuyaseg-controller/src/reporting/tracing_evt.rs
View file

@ -1,5 +1,4 @@
use std::sync::Arc; use std::sync::Arc;
use std::sync::nonpoison::RwLock;
use k8s_openapi::api::core::v1::ObjectReference; use k8s_openapi::api::core::v1::ObjectReference;
use kube::runtime::events::{ use kube::runtime::events::{
@ -46,34 +45,21 @@ pub struct TracingK8sEventLayerConfig {
/// ///
/// - k8s native events you must record yourself - thats what `tx` is for. /// - k8s native events you must record yourself - thats what `tx` is for.
/// - prometheus metrics are written automatically (if metrics set), but labels are also send with `tx`. /// - prometheus metrics are written automatically (if metrics set), but labels are also send with `tx`.
/// - k8s events are NOT rate-limited. K8s API rejects >10 events/object/min. /// - k8s events are rate-limited to 10 events/min
/// Channel consumer must batch/dedup to avoid API errors.
pub struct TracingK8sEventLayer { pub struct TracingK8sEventLayer {
k8s_tx: RwLock<Option<Sender<Arc<K8sEvent>>>>, k8s_tx: Option<Sender<Arc<K8sEvent>>>,
pro_tx: RwLock<Option<Sender<Arc<ProLabel>>>>, pro_tx: Option<Sender<Arc<ProLabel>>>,
otel_tx: RwLock<Option<Sender<Arc<OtelObservation>>>>, otel_tx: Option<Sender<Arc<OtelObservation>>>,
} }
impl TracingK8sEventLayer { impl TracingK8sEventLayer {
pub fn new(config: TracingK8sEventLayerConfig) -> Self { pub fn new(config: TracingK8sEventLayerConfig) -> Self {
Self { Self {
k8s_tx: RwLock::new(config.k8s_events), k8s_tx: config.k8s_events,
pro_tx: RwLock::new(config.prometheus), pro_tx: config.prometheus,
otel_tx: RwLock::new(config.otel), otel_tx: config.otel,
} }
} }
pub fn reload_k8s(&self, tx: Option<Sender<Arc<K8sEvent>>>) {
*self.k8s_tx.write() = tx;
}
pub fn reload_pro(&self, tx: Option<Sender<Arc<ProLabel>>>) {
*self.pro_tx.write() = tx;
}
pub fn reload_otel(&self, tx: Option<Sender<Arc<OtelObservation>>>) {
*self.otel_tx.write() = tx;
}
} }
impl<S> Layer<S> for TracingK8sEventLayer impl<S> Layer<S> for TracingK8sEventLayer
@ -152,7 +138,7 @@ where
if event_fields.no_event.is_none_or(|v| !v) if event_fields.no_event.is_none_or(|v| !v)
&& span_fields.no_event.is_none_or(|v| !v) && span_fields.no_event.is_none_or(|v| !v)
&& let Some(event) = event_fields.try_to_event(k8s_event_type) && let Some(event) = event_fields.try_to_event(k8s_event_type)
&& let Some(tx) = self.k8s_tx.read().clone() && let Some(tx) = self.k8s_tx.as_ref()
{ {
_ = tx _ = tx
.send(Arc::new(K8sEvent { .send(Arc::new(K8sEvent {
@ -178,7 +164,7 @@ where
if event_fields.no_pro.is_none_or(|v| !v) if event_fields.no_pro.is_none_or(|v| !v)
&& span_fields.no_pro.is_none_or(|v| !v) && span_fields.no_pro.is_none_or(|v| !v)
&& let Some(label) = try_create_pro_label(&span_fields, &event_fields, pro_severity, str_trace_id.clone()) && let Some(label) = try_create_pro_label(&span_fields, &event_fields, pro_severity, str_trace_id.clone())
&& let Some(tx) = self.pro_tx.read().clone() && let Some(tx) = self.pro_tx.as_ref()
{ {
_ = tx.send(Arc::new(label)).inspect_err( _ = tx.send(Arc::new(label)).inspect_err(
|e| tracing::error!(debug=?e, "Failed to issue event on prometheus label measure from tracing"), |e| tracing::error!(debug=?e, "Failed to issue event on prometheus label measure from tracing"),
@ -200,7 +186,7 @@ where
}; };
if event_fields.no_otel.is_none_or(|v| !v) if event_fields.no_otel.is_none_or(|v| !v)
&& span_fields.no_otel.is_none_or(|v| !v) && span_fields.no_otel.is_none_or(|v| !v)
&& let Some(tx) = self.otel_tx.read().clone() && let Some(tx) = self.otel_tx.as_ref()
{ {
_ = tx _ = tx
.send(Arc::new(OtelObservation::Log { .send(Arc::new(OtelObservation::Log {