Trez.One
cbdc8f35a5
Gitea Branch PR & Ansible Deployment / Check and Create PR (push) Successful in 8m53s
Gitea Branch PR & Ansible Deployment / Docker Compose & Ansible Lints (push) Failing after 12m35s
Gitea Branch PR & Ansible Deployment / PR Merge (push) Successful in 31s
Gitea Branch PR & Ansible Deployment / Ansible Config Deployment (push) Failing after 19m9s
404 lines
14 KiB
Plaintext
404 lines
14 KiB
Plaintext
{% set vault_addr = 'https://vault.trez.wtf' %}
|
|
{% set secrets_path = 'rinoa-docker/env' %}
|
|
|
|
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
// Agent globals
|
|
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
local.file "endpoints" {
|
|
// The endpoints file is used to define the endpoints, credentials and options
|
|
// for the Agent export to.
|
|
filename = "/etc/alloy/endpoints.json"
|
|
}
|
|
|
|
discovery.docker "rinoadocker" {
|
|
host = env("DOCKER_HOST")
|
|
}
|
|
|
|
tracing {
|
|
write_to = [otelcol.exporter.otlp.tempo.input]
|
|
}
|
|
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
// Metrics
|
|
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
prometheus.remote_write "mimir" {
|
|
endpoint {
|
|
url = json_path(local.file.endpoints.content, ".metrics.url")[0]
|
|
basic_auth {
|
|
username = json_path(local.file.endpoints.content, ".metrics.basicAuth.username")[0]
|
|
password = json_path(local.file.endpoints.content, ".metrics.basicAuth.password")[0]
|
|
}
|
|
}
|
|
}
|
|
|
|
prometheus.scrape "prometheus" {
|
|
targets = [{
|
|
__address__ = "localhost:12345",
|
|
}]
|
|
forward_to = [prometheus.remote_write.mimir.receiver]
|
|
job_name = "prometheus"
|
|
}
|
|
|
|
prometheus.exporter.unix "rinoa" {
|
|
procfs_path = "/host/proc"
|
|
sysfs_path = "/host/sys"
|
|
rootfs_path = "/rootfs"
|
|
}
|
|
|
|
prometheus.scrape "rinoa" {
|
|
targets = prometheus.exporter.unix.rinoa.targets
|
|
forward_to = [prometheus.remote_write.mimir.receiver]
|
|
job_name = "rinoa_host"
|
|
}
|
|
|
|
prometheus.exporter.cadvisor "docker" {
|
|
docker_host = env("DOCKER_HOST")
|
|
storage_duration = "5m"
|
|
}
|
|
|
|
prometheus.scrape "docker" {
|
|
targets = prometheus.exporter.cadvisor.docker.targets
|
|
forward_to = [prometheus.remote_write.mimir.receiver]
|
|
job_name = "docker_stats"
|
|
}
|
|
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
// Logging
|
|
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
loki.write "loki" {
|
|
endpoint {
|
|
url = json_path(local.file.endpoints.content, ".logs.url")[0]
|
|
basic_auth {
|
|
username = json_path(local.file.endpoints.content, ".logs.basicAuth.username")[0]
|
|
password = json_path(local.file.endpoints.content, ".logs.basicAuth.password")[0]
|
|
}
|
|
}
|
|
external_labels = {}
|
|
}
|
|
|
|
loki.source.journal "hostjournal" {
|
|
forward_to = [loki.write.loki.receiver]
|
|
max_age = "24h"
|
|
path = "/rootfs/var/log/journal/"
|
|
labels = {
|
|
job = "host-journal",
|
|
}
|
|
}
|
|
|
|
local.file_match "system" {
|
|
path_targets = [{
|
|
__address__ = "localhost",
|
|
__path__ = "/rootfs/var/log/*log",
|
|
job = "varlogs",
|
|
}]
|
|
}
|
|
|
|
loki.source.file "system" {
|
|
targets = local.file_match.system.targets
|
|
forward_to = [loki.write.loki.receiver]
|
|
}
|
|
|
|
loki.source.docker "containers" {
|
|
host = env("DOCKER_HOST")
|
|
targets = discovery.docker.rinoadocker.targets
|
|
forward_to = [loki.write.loki.receiver]
|
|
labels = {
|
|
job = "containerlogs",
|
|
}
|
|
}
|
|
|
|
loki.process "containers" {
|
|
forward_to = [loki.write.loki.receiver]
|
|
// stage.docker {}
|
|
stage.json {
|
|
expressions = {
|
|
attrs = "",
|
|
output = "log",
|
|
stream = "stream",
|
|
}
|
|
}
|
|
|
|
stage.json {
|
|
expressions = {
|
|
tag = "",
|
|
}
|
|
source = "attrs"
|
|
}
|
|
|
|
stage.regex {
|
|
expression = "(?P<image_name>(?:[^|]*[^|])).(?P<container_name>(?:[^|]*[^|])).(?P<image_id>(?:[^|]*[^|])).(?P<container_id>(?:[^|]*[^|]))"
|
|
source = "tag"
|
|
}
|
|
|
|
stage.timestamp {
|
|
source = "time"
|
|
format = "RFC3339Nano"
|
|
}
|
|
|
|
stage.labels {
|
|
values = {
|
|
container_id = null,
|
|
container_name = null,
|
|
image_id = null,
|
|
image_name = null,
|
|
stream = null,
|
|
tag = null,
|
|
}
|
|
}
|
|
|
|
stage.output {
|
|
source = "output"
|
|
}
|
|
}
|
|
|
|
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
// Traces
|
|
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
beyla.ebpf "rinoadocker" {
|
|
open_port = "80-65535"
|
|
routes {
|
|
unmatched = "heauristic"
|
|
}
|
|
output {
|
|
traces = [
|
|
otelcol.connector.servicegraph.tracemetrics.input,
|
|
otelcol.connector.spanmetrics.tracemetrics.input,
|
|
otelcol.processor.batch.default.input,
|
|
otelcol.connector.spanlogs.autologging.input,
|
|
]
|
|
}
|
|
}
|
|
|
|
prometheus.scrape "beyla" {
|
|
targets = beyla.ebpf.rinoadocker.targets
|
|
forward_to = [prometheus.remote_write.mimir.receiver]
|
|
}
|
|
|
|
otelcol.auth.headers "tempo" {
|
|
header {
|
|
key = "Authorization"
|
|
value = join(["Basic ", json_path(local.file.endpoints.content, ".traces.basicAuthToken")[0]], "")
|
|
}
|
|
}
|
|
|
|
otelcol.processor.batch "default" {
|
|
// Wait until we've received 16K of data.
|
|
send_batch_size = 16384
|
|
send_batch_max_size = 16384
|
|
// Or until 2 seconds have elapsed.
|
|
timeout = "2s"
|
|
// When the Agent has enough batched data, send it to the OpenTelemetry exporter named 'tempo'.
|
|
output {
|
|
traces = [otelcol.exporter.otlp.tempo.input]
|
|
}
|
|
}
|
|
|
|
otelcol.exporter.otlp "tempo" {
|
|
// Define the client for exporting.
|
|
client {
|
|
// Authentication block.
|
|
auth = otelcol.auth.headers.tempo.handler
|
|
|
|
// Send to the locally running Tempo instance, on port 4317 (OTLP gRPC).
|
|
endpoint = json_path(local.file.endpoints.content, ".traces.url")[0]
|
|
|
|
// Configure TLS settings for communicating with the endpoint.
|
|
tls {
|
|
// The connection is insecure.
|
|
insecure = json_path(local.file.endpoints.content, ".traces.tls.insecure")[0]
|
|
// Do not verify TLS certificates when connecting.
|
|
insecure_skip_verify = json_path(local.file.endpoints.content, ".traces.tls.insecureSkipVerify")[0]
|
|
}
|
|
}
|
|
}
|
|
|
|
otelcol.connector.spanlogs "autologging" {
|
|
// We only want to output a line for each root span (ie. every single trace), and not for every
|
|
// process or span (outputting a line for every span would be extremely verbose).
|
|
spans = false
|
|
roots = true
|
|
processes = false
|
|
// We want to ensure that the following three span attributes are included in the log line, if
|
|
// present.
|
|
span_attributes = [ "http.method", "http.target", "http.status_code" ]
|
|
|
|
// Overrides the default key in the log line to be `traceId`, which is then used by Grafana to
|
|
// identify the trace ID for correlation with the Tempo datasource.
|
|
overrides {
|
|
trace_id_key = "traceId"
|
|
}
|
|
// Send to the OpenTelemetry Loki exporter.
|
|
output {
|
|
logs = [otelcol.exporter.loki.autologging.input]
|
|
}
|
|
}
|
|
|
|
// Simply forwards the incoming OpenTelemetry log format out as a Loki log.
|
|
// We need this stage to ensure we can then process the logline as a Loki object.
|
|
otelcol.exporter.loki "autologging" {
|
|
forward_to = [loki.process.autologging.receiver]
|
|
}
|
|
|
|
// The Loki processor allows us to accept a correctly formatted Loki log and mutate it into
|
|
// a set of fields for output.
|
|
loki.process "autologging" {
|
|
// The JSON stage simply extracts the `body` (the actual logline) from the Loki log, ignoring
|
|
// all other fields.
|
|
stage.json {
|
|
expressions = { "body" = "" }
|
|
}
|
|
// The output stage takes the body (the main logline) and uses this as the source for the output
|
|
// logline. In this case, it essentially turns it into logfmt.
|
|
stage.output {
|
|
source = "body"
|
|
}
|
|
|
|
// Finally send the processed logline onto the Loki exporter.
|
|
forward_to = [loki.write.autologging.receiver]
|
|
}
|
|
|
|
// The Loki writer receives a processed Loki log and then writes it to a Loki instance.
|
|
loki.write "autologging" {
|
|
// Add the `agent` value to the `job` label, so we can identify it as having been generated
|
|
// by Grafana Agent when querying.
|
|
external_labels = {
|
|
job = "agent",
|
|
}
|
|
|
|
// Output the Loki log to the local Loki instance.
|
|
endpoint {
|
|
url = json_path(local.file.endpoints.content, ".logs.url")[0]
|
|
|
|
// The basic auth credentials for the Loki instance.
|
|
basic_auth {
|
|
username = json_path(local.file.endpoints.content, ".logs.basicAuth.username")[0]
|
|
password = json_path(local.file.endpoints.content, ".logs.basicAuth.password")[0]
|
|
}
|
|
}
|
|
}
|
|
|
|
// The Tail Sampling processor will use a set of policies to determine which received traces to keep
|
|
// and send to Tempo.
|
|
otelcol.processor.tail_sampling "errors" {
|
|
// Total wait time from the start of a trace before making a sampling decision. Note that smaller time
|
|
// periods can potentially cause a decision to be made before the end of a trace has occurred.
|
|
decision_wait = "30s"
|
|
|
|
// The following policies follow a logical OR pattern, meaning that if any of the policies match,
|
|
// the trace will be kept. For logical AND, you can use the `and` policy. Every span of a trace is
|
|
// examined by each policy in turn. A match will cause a short-circuit.
|
|
|
|
// This policy defines that traces that contain errors should be kept.
|
|
policy {
|
|
// The name of the policy can be used for logging purposes.
|
|
name = "sample-erroring-traces"
|
|
// The type must match the type of policy to be used, in this case examing the status code
|
|
// of every span in the trace.
|
|
type = "status_code"
|
|
// This block determines the error codes that should match in order to keep the trace,
|
|
// in this case the OpenTelemetry 'ERROR' code.
|
|
status_code {
|
|
status_codes = [ "ERROR" ]
|
|
}
|
|
}
|
|
|
|
// This policy defines that only traces that are longer than 200ms in total should be kept.
|
|
policy {
|
|
// The name of the policy can be used for logging purposes.
|
|
name = "sample-long-traces"
|
|
// The type must match the policy to be used, in this case the total latency of the trace.
|
|
type = "latency"
|
|
// This block determines the total length of the trace in milliseconds.
|
|
latency {
|
|
threshold_ms = 200
|
|
}
|
|
}
|
|
|
|
// The output block forwards the kept traces onto the batch processor, which will marshall them
|
|
// for exporting to Tempo.
|
|
output {
|
|
traces = [otelcol.processor.batch.default.input]
|
|
}
|
|
}
|
|
|
|
// The Spanmetrics Connector will generate RED metrics based on the incoming trace span data.
|
|
otelcol.connector.spanmetrics "tracemetrics" {
|
|
// The namespace explicit adds a prefix to all the generated span metrics names.
|
|
// In this case, we'll ensure they match as closely as possible those generated by Tempo.
|
|
namespace = "traces.spanmetrics"
|
|
|
|
// Each extra dimension (metrics label) to be added to the generated metrics from matching span attributes. These
|
|
// need to be defined with a name and optionally a default value (in the following cases, we do not want a default
|
|
// value if the span attribute is not present).
|
|
dimension {
|
|
name = "http.method"
|
|
}
|
|
dimension {
|
|
name = "http.target"
|
|
}
|
|
dimension {
|
|
name = "http.status_code"
|
|
}
|
|
dimension {
|
|
name = "service.version"
|
|
}
|
|
|
|
// A histogram block must be present, either explicitly defining bucket values or via an exponential block.
|
|
// We do the latter here.
|
|
histogram {
|
|
explicit {
|
|
}
|
|
}
|
|
|
|
// The exemplar block is added to ensure we generate exemplars for traces on relevant metric values.
|
|
exemplars {
|
|
enabled = true
|
|
}
|
|
|
|
// Generated metrics data is in OTLP format. We send this data to the OpenTelemetry Prometheus exporter to ensure
|
|
// it gets transformed into Prometheus format data.
|
|
output {
|
|
metrics = [otelcol.exporter.prometheus.tracemetrics.input]
|
|
}
|
|
}
|
|
|
|
// The Servicegraph Connector will generate service graph metrics (edges and nodes) based on incoming trace spans.
|
|
otelcol.connector.servicegraph "tracemetrics" {
|
|
// Extra dimensions (metrics labels) to be added to the generated metrics from matching span attributes.
|
|
// For this component, this is defined as an array. There are no default values and the labels will not be generated
|
|
// for missing span attributes.
|
|
dimensions = [
|
|
"http.method",
|
|
"http.target",
|
|
"http.status_code",
|
|
"service.version",
|
|
]
|
|
|
|
// Generated metrics data is in OTLP format. We send this data to the OpenTelemetry Prometheus exporter to ensure
|
|
// it gets transformed into Prometheus format data.
|
|
output {
|
|
metrics = [otelcol.exporter.prometheus.tracemetrics.input]
|
|
}
|
|
}
|
|
|
|
otelcol.exporter.prometheus "tracemetrics" {
|
|
// Forward to our local Prometheus remote writer which will send the metrics to Mimir.
|
|
forward_to = [prometheus.remote_write.mimir.receiver]
|
|
}
|
|
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
// Profiling
|
|
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
pyroscope.write "pyroscope" {
|
|
endpoint {
|
|
url = json_path(local.file.endpoints.content, ".profiles.url")[0]
|
|
basic_auth {
|
|
username = json_path(local.file.endpoints.content, ".profiles.basicAuth.username")[0]
|
|
password = json_path(local.file.endpoints.content, ".profiles.basicAuth.password")[0]
|
|
}
|
|
}
|
|
external_labels = {}
|
|
}
|
|
|
|
pyroscope.ebpf "rinoadocker" {
|
|
forward_to = [pyroscope.write.pyroscope.receiver]
|
|
targets = discovery.docker.rinoadocker.targets
|
|
} |