kubeworks/kube-yaseg
Watch
1
0
Fork
You've already forked kube-yaseg
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller/reporting): rate limit k8s events & add external cancellation

Browse source
This commit is contained in:
spushk 2026-05-16 04:49:34 +05:00 • committed by spushk
commit a42cc2e7ac
Signed by: spushk
GPG key ID: 7797B761E105E8AA
1 changed files with 124 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

161
kuyaseg-controller/src/reporting/mod.rs
View file

@ -1,3 +1,4 @@
use std::collections::VecDeque;
use std::sync::Arc;
use std::time::Duration;
@ -49,7 +50,7 @@ pub struct ReportingInit {
pub handle: JoinSet<()>,
}
pub fn init(k8s_client: kube::Client) -> anyhow::Result<ReportingInit> {
pub fn init(k8s_client: kube::Client, ct: CancellationToken) -> anyhow::Result<ReportingInit> {
let config = ReportingConfig::builder()
.env()
.load()
@ -59,9 +60,14 @@ pub fn init(k8s_client: kube::Client) -> anyhow::Result<ReportingInit> {
let k8s_tx = config
.k8s_events
.then(|| init_k8s_events(k8s_client, &mut reporting_jobs));
let pro_tx = config.prometheus.then(|| init_pro_metrics(&mut reporting_jobs));
let otel_tx = config.opentelemetry.then(|| init_otel(&mut reporting_jobs));
.then(|| init_k8s_events(k8s_client, &mut reporting_jobs, ct.clone()));
let pro_tx = config
.prometheus
.then(|| init_pro_metrics(&mut reporting_jobs, ct.clone()));
let otel_tx = config
.opentelemetry
.then(|| init_otel(&mut reporting_jobs, ct.clone()))
.transpose()?;
Ok(ReportingInit {
tracing_layer: TracingK8sEventLayer::new(TracingK8sEventLayerConfig {
@ -73,7 +79,7 @@ pub fn init(k8s_client: kube::Client) -> anyhow::Result<ReportingInit> {
})
}
fn init_k8s_events(client: kube::Client, joinset: &mut JoinSet<()>) -> Sender<Arc<K8sEvent>> {
fn init_k8s_events(client: kube::Client, joinset: &mut JoinSet<()>, ct: CancellationToken) -> Sender<Arc<K8sEvent>> {
use kube::runtime::events::{
Recorder,
Reporter,
@ -81,7 +87,10 @@ fn init_k8s_events(client: kube::Client, joinset: &mut JoinSet<()>) -> Sender<Ar
use crate::reporting::k8s_event::K8sEventRecorder;
let (tx, mut rx) = broadcast::channel::<Arc<K8sEvent>>(128);
const WARN_THRESHOLD: usize = 10;
const MAX_CAPACITY: usize = 100;
let (tx, mut rx) = broadcast::channel::<Arc<K8sEvent>>(512);
let reporter = Reporter {
controller: env!("CARGO_PKG_NAME").into(),
instance: std::env::var("CONTROLLER_POD_NAME").ok(),
@ -91,36 +100,65 @@ fn init_k8s_events(client: kube::Client, joinset: &mut JoinSet<()>) -> Sender<Ar
let rtx = tx.clone(); // for dumb recovery
joinset.spawn(async move {
let mut queue: VecDeque<Arc<K8sEvent>> = VecDeque::new();
let mut warned = false;
let mut drain_tick = tokio::time::interval(Duration::from_secs(6)); // 10 event/min
let mut rec_limit = 4;
while rec_limit > 0 {
match rx.recv().await {
Ok(ev) => {
recorder.record(&ev).await;
rec_limit = 4;
}
Err(e) => {
tracing::error!(no_event=true, debug=?e, "Failed to receive k8s event from tracing");
rx = rtx.subscribe(); // dumb recover strategy
tokio::time::sleep(Duration::from_secs(3)).await;
rec_limit -= 1;
tokio::select! {
biased;
_ = ct.cancelled() => break,
_ = drain_tick.tick() => {
if let Some(ev) = queue.pop_front() {
recorder.record(&ev).await;
}
},
msg = rx.recv() => {
match msg {
Ok(ev) => {
if queue.is_empty() {
recorder.record(&ev).await;
warned = false;
} else {
if queue.len() >= MAX_CAPACITY {
queue.pop_front();
}
queue.push_back(ev);
if queue.len() > WARN_THRESHOLD && !warned {
tracing::warn!(no_event=true, queue_len=queue.len(), "K8s event queue exceeded threshold");
warned = true;
} else if queue.len() <= WARN_THRESHOLD {
warned = false;
}
}
rec_limit = 4;
}
Err(e) => {
tracing::error!(no_event=true, debug=?e, "Failed to receive k8s event from tracing");
rx = rtx.subscribe(); // dumb recover strategy
tokio::time::sleep(Duration::from_secs(3)).await;
rec_limit -= 1;
}
}
}
}
}
tracing::error!(
no_event = true,
"K8s Event reporting from tracing stopped from limit reached"
"K8s Event reporting from tracing stopped"
);
});
tx
}
fn init_pro_metrics(joinset: &mut JoinSet<()>) -> Sender<Arc<ProLabel>> {
fn init_pro_metrics(joinset: &mut JoinSet<()>, ct: CancellationToken) -> Sender<Arc<ProLabel>> {
use prometheus_client::registry::Registry;
use crate::reporting::prometheus::ProMetrics;
let (tx, mut rx) = broadcast::channel::<Arc<ProLabel>>(128);
let (tx, mut rx) = broadcast::channel::<Arc<ProLabel>>(512);
let metrics = ProMetrics::new();
let registration_cancellation = CancellationToken::new();
@ -128,28 +166,37 @@ fn init_pro_metrics(joinset: &mut JoinSet<()>) -> Sender<Arc<ProLabel>> {
let rtx = tx.clone();
let ctk = registration_cancellation.clone();
let promt = metrics.clone();
let consumer_ct = ct.clone();
joinset.spawn(async move {
let mut rec_limit = 4;
while rec_limit > 0 {
match rx.recv().await {
Ok(met) => {
promt.measure(&met);
}
Err(e) => {
tracing::error!(no_pro=true, debug=?e, "Failed to receive prometheus measurement from tracing");
rx = rtx.subscribe(); // dumb recover strategy;
tokio::time::sleep(Duration::from_secs(3)).await;
rec_limit -= 1;
tokio::select! {
biased;
_ = consumer_ct.cancelled() => break,
msg = rx.recv() => {
match msg {
Ok(met) => {
promt.measure(&met);
rec_limit = 4;
}
Err(e) => {
tracing::error!(no_pro=true, debug=?e, "Failed to receive prometheus measurement from tracing");
rx = rtx.subscribe(); // dumb recover strategy;
tokio::time::sleep(Duration::from_secs(3)).await;
rec_limit -= 1;
}
}
}
}
}
ctk.cancel();
tracing::error!(
no_pro = true,
"Prometheus measurement from tracing stopped from limit reached"
"Prometheus measurement from tracing stopped"
);
});
let ctk = ct.clone();
joinset.spawn(async move {
let mut registry = Registry::default();
loop {
@ -158,7 +205,11 @@ fn init_pro_metrics(joinset: &mut JoinSet<()>) -> Sender<Arc<ProLabel>> {
metrics.register(&mut registry);
},
_ = registration_cancellation.cancelled() => {
tracing::warn!(no_pro=true, "Stopped publishing prometheus metrics: job cancelled");
tracing::warn!(no_pro=true, "Stopped publishing prometheus metrics: consumer job cancelled");
break;
}
_ = ctk.cancelled() => {
tracing::warn!(no_pro=true, "Stopped publishing prometheus metrics: shutdown");
break;
}
}
@ -168,48 +219,52 @@ fn init_pro_metrics(joinset: &mut JoinSet<()>) -> Sender<Arc<ProLabel>> {
tx
}
fn init_otel(joinset: &mut JoinSet<()>) -> Sender<Arc<OtelObservation>> {
fn init_otel(joinset: &mut JoinSet<()>, ct: CancellationToken) -> anyhow::Result<Sender<Arc<OtelObservation>>> {
use crate::reporting::otel::OtelAggregator;
let exporter = opentelemetry_otlp::LogExporter::builder()
.with_tonic()
.build()
.expect("Failed to create OTLP log exporter");
.context("Failed to build OTLP log exporter")?;
let provider = opentelemetry_sdk::logs::SdkLoggerProvider::builder()
.with_batch_exporter(exporter)
.build();
let aggregator = OtelAggregator::new(provider);
let aggregator = Arc::new(OtelAggregator::new(provider));
let task_aggregator = aggregator.clone();
let (tx, mut rx) = broadcast::channel::<Arc<OtelObservation>>(128);
let (tx, mut rx) = broadcast::channel::<Arc<OtelObservation>>(512);
let rtx = tx.clone();
joinset.spawn(async move {
let mut rec_limit = 4;
while rec_limit > 0 {
match rx.recv().await {
Ok(obs) => {
aggregator.observe(&obs);
rec_limit = 4;
}
Err(e) => {
tracing::error!(
no_otel = true,
debug = ?e,
"Failed to receive otel observation from tracing"
);
rx = rtx.subscribe();
tokio::time::sleep(Duration::from_secs(3)).await;
rec_limit -= 1;
tokio::select! {
biased;
_ = ct.cancelled() => break,
msg = rx.recv() => {
match msg {
Ok(obs) => {
task_aggregator.observe(&obs);
rec_limit = 4;
}
Err(e) => {
tracing::error!(
no_otel = true,
debug = ?e,
"Failed to receive otel observation from tracing"
);
rx = rtx.subscribe();
tokio::time::sleep(Duration::from_secs(3)).await;
rec_limit -= 1;
}
}
}
}
}
tracing::error!(
no_otel = true,
"OpenTelemetry reporting from tracing stopped from limit reached"
);
tracing::error!(no_otel = true, "OpenTelemetry reporting from tracing stopped");
});
tx
Ok(tx)
}