kubeworks/kube-yaseg
Watch
1
0
Fork
You've already forked kube-yaseg
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(reporting)!: drop OTEL (for now) & rename layer field names & fix prometheus registry

Browse source
This commit is contained in:
spushk 2026-05-30 10:19:53 +05:00 • committed by spushk
commit e7f6ddf741
Signed by: spushk
GPG key ID: 7797B761E105E8AA
1 changed files with 76 additions and 289 deletions
Download patch file Download diff file Expand all files Collapse all files

134
kuyaseg-controller/src/reporting/mod.rs
View file

@ -4,6 +4,7 @@ use std::time::Duration;
use anyhow::Context;
use confique::Config;
use opentelemetry::TraceId;
use tokio::sync::broadcast::{
self,
Sender,
@ -11,19 +12,27 @@ use tokio::sync::broadcast::{
use tokio::task::JoinSet;
use tokio_util::sync::CancellationToken;
use crate::reporting::k8s_event::K8sEvent;
use crate::reporting::otel::OtelObservation;
use crate::reporting::k8s::K8sEvent;
use crate::reporting::prometheus::ProLabel;
use crate::reporting::tracing_evt::{
TracingK8sEventLayer,
TracingK8sEventLayerConfig,
};
pub mod k8s_event;
pub mod otel;
pub mod k8s;
pub mod prometheus;
pub mod tracing_evt;
pub fn span_trace_id() -> Option<TraceId> {
use opentelemetry::trace::TraceContextExt;
use tracing_opentelemetry::OpenTelemetrySpanExt;
match tracing::Span::current().context().span().span_context().trace_id() {
TraceId::INVALID => None,
other => Some(other),
}
}
#[derive(Config)]
pub struct ReportingConfig {
#[config(env = "KUYASEG_REPORT_K8S")]
@ -32,22 +41,13 @@ pub struct ReportingConfig {
#[config(env = "KUYASEG_REPORT_PROMETHEUS")]
#[config(default = false)]
pub prometheus: bool,
#[config(env = "KUYASEG_REPORT_OTEL")]
#[config(default = false)]
pub opentelemetry: bool,
}
pub fn trace_id_str(trace_id: &opentelemetry::TraceId) -> Option<String> {
match trace_id {
&opentelemetry::TraceId::INVALID => None,
x => Some(x.to_string()),
}
}
/// You should await on handle somewhere
pub struct ReportingInit {
pub tracing_layer: TracingK8sEventLayer,
pub handle: JoinSet<()>,
pub pro_registry: Option<prometheus_client::registry::Registry>,
}
pub fn init(k8s_client: kube::Client, ct: CancellationToken) -> anyhow::Result<ReportingInit> {
@ -61,21 +61,18 @@ pub fn init(k8s_client: kube::Client, ct: CancellationToken) -> anyhow::Result<R
let k8s_tx = config
.k8s_events
.then(|| init_k8s_events(k8s_client, &mut reporting_jobs, ct.clone()));
let pro_tx = config
let (pro_tx, pro_registry) = config
.prometheus
.then(|| init_pro_metrics(&mut reporting_jobs, ct.clone()));
let otel_tx = config
.opentelemetry
.then(|| init_otel(&mut reporting_jobs, ct.clone()))
.transpose()?;
.then(|| init_pro_metrics(&mut reporting_jobs, ct.clone()))
.unzip();
Ok(ReportingInit {
tracing_layer: TracingK8sEventLayer::new(TracingK8sEventLayerConfig {
k8s_events: k8s_tx,
prometheus: pro_tx,
otel: otel_tx,
}),
handle: reporting_jobs,
pro_registry,
})
}
@ -85,7 +82,7 @@ fn init_k8s_events(client: kube::Client, joinset: &mut JoinSet<()>, ct: Cancella
Reporter,
};
use crate::reporting::k8s_event::K8sEventRecorder;
use crate::reporting::k8s::K8sEventRecorder;
const WARN_THRESHOLD: usize = 10;
const MAX_CAPACITY: usize = 100;
@ -126,7 +123,7 @@ fn init_k8s_events(client: kube::Client, joinset: &mut JoinSet<()>, ct: Cancella
}
queue.push_back(ev);
if queue.len() > WARN_THRESHOLD && !warned {
tracing::warn!(k8s_no_event=true, queue_len=queue.len(), "K8s event queue exceeded threshold");
tracing::warn!(k13t.no_report=true, debug=?queue.len(), "K8s event queue exceeded threshold");
warned = true;
} else if queue.len() <= WARN_THRESHOLD {
warned = false;
@ -135,7 +132,7 @@ fn init_k8s_events(client: kube::Client, joinset: &mut JoinSet<()>, ct: Cancella
rec_limit = 4;
}
Err(e) => {
tracing::error!(k8s_no_event=true, debug=?e, "Failed to receive k8s event from tracing");
tracing::error!(k13t.no_report=true, debug=?e, "Failed to receive k8s event from tracing");
rx = rtx.subscribe(); // dumb recover strategy
tokio::time::sleep(Duration::from_secs(3)).await;
rec_limit -= 1;
@ -145,7 +142,7 @@ fn init_k8s_events(client: kube::Client, joinset: &mut JoinSet<()>, ct: Cancella
}
}
tracing::error!(
no_event = true,
k13t.no_report = true,
"K8s Event reporting from tracing stopped"
);
});
@ -153,7 +150,10 @@ fn init_k8s_events(client: kube::Client, joinset: &mut JoinSet<()>, ct: Cancella
tx
}
fn init_pro_metrics(joinset: &mut JoinSet<()>, ct: CancellationToken) -> Sender<Arc<ProLabel>> {
fn init_pro_metrics(
joinset: &mut JoinSet<()>,
ct: CancellationToken,
) -> (Sender<Arc<ProLabel>>, prometheus_client::registry::Registry) {
use prometheus_client::registry::Registry;
use crate::reporting::prometheus::ProMetrics;
@ -161,10 +161,7 @@ fn init_pro_metrics(joinset: &mut JoinSet<()>, ct: CancellationToken) -> Sender<
let (tx, mut rx) = broadcast::channel::<Arc<ProLabel>>(512);
let metrics = ProMetrics::new();
let registration_cancellation = CancellationToken::new();
let rtx = tx.clone();
let ctk = registration_cancellation.clone();
let promt = metrics.clone();
let consumer_ct = ct.clone();
joinset.spawn(async move {
@ -180,7 +177,7 @@ fn init_pro_metrics(joinset: &mut JoinSet<()>, ct: CancellationToken) -> Sender<
rec_limit = 4;
}
Err(e) => {
tracing::error!(k8s_no_pro=true, debug=?e, "Failed to receive prometheus measurement from tracing");
tracing::error!(k13t.no_report=true, debug=?e, "Failed to receive prometheus measurement from tracing");
rx = rtx.subscribe(); // dumb recover strategy;
tokio::time::sleep(Duration::from_secs(3)).await;
rec_limit -= 1;
@ -189,82 +186,17 @@ fn init_pro_metrics(joinset: &mut JoinSet<()>, ct: CancellationToken) -> Sender<
}
}
}
ctk.cancel();
tracing::error!(
no_pro = true,
k13t.no_report=true,
"Prometheus measurement from tracing stopped"
);
});
let ctk = ct.clone();
joinset.spawn(async move {
let registry = {
let mut registry = Registry::default();
loop {
tokio::select! {
_ = tokio::time::sleep(Duration::from_secs(10)) => {
metrics.register(&mut registry);
},
_ = registration_cancellation.cancelled() => {
tracing::warn!(k8s_no_pro=true, "Stopped publishing prometheus metrics: consumer job cancelled");
break;
}
_ = ctk.cancelled() => {
tracing::warn!(k8s_no_pro=true, "Stopped publishing prometheus metrics: shutdown");
break;
}
}
}
});
metrics.register(&mut registry);
registry
};
tx
}
fn init_otel(joinset: &mut JoinSet<()>, ct: CancellationToken) -> anyhow::Result<Sender<Arc<OtelObservation>>> {
use crate::reporting::otel::OtelAggregator;
let exporter = opentelemetry_otlp::LogExporter::builder()
.with_tonic()
.build()
.context("Failed to build OTLP log exporter")?;
let provider = opentelemetry_sdk::logs::SdkLoggerProvider::builder()
.with_batch_exporter(exporter)
.build();
let aggregator = Arc::new(OtelAggregator::new(provider));
let task_aggregator = aggregator.clone();
let (tx, mut rx) = broadcast::channel::<Arc<OtelObservation>>(512);
let rtx = tx.clone();
joinset.spawn(async move {
let mut rec_limit = 4;
while rec_limit > 0 {
tokio::select! {
biased;
_ = ct.cancelled() => break,
msg = rx.recv() => {
match msg {
Ok(obs) => {
task_aggregator.observe(&obs);
rec_limit = 4;
}
Err(e) => {
tracing::error!(
k8s_no_otel = true,
debug = ?e,
"Failed to receive otel observation from tracing"
);
rx = rtx.subscribe();
tokio::time::sleep(Duration::from_secs(3)).await;
rec_limit -= 1;
}
}
}
}
}
tracing::error!(k8s_no_otel = true, "OpenTelemetry reporting from tracing stopped");
});
Ok(tx)
(tx, registry)
}