diff --git a/app-configs/rinoa/signoz/common/otel/otel-collector-config.yaml.j2 b/app-configs/rinoa/signoz/common/otel/otel-collector-config.yaml.j2 index 714f8b2..8a835b7 100644 --- a/app-configs/rinoa/signoz/common/otel/otel-collector-config.yaml.j2 +++ b/app-configs/rinoa/signoz/common/otel/otel-collector-config.yaml.j2 @@ -2,14 +2,16 @@ receivers: httplogreceiver/json: endpoint: 0.0.0.0:8082 source: json + otlp: protocols: grpc: endpoint: 0.0.0.0:4317 http: endpoint: 0.0.0.0:4318 + hostmetrics: - collection_interval: 60s # Frequency of metrics collection. + collection_interval: 60s scrapers: cpu: {} load: {} @@ -17,6 +19,7 @@ receivers: disk: {} filesystem: {} network: {} + docker_stats: endpoint: unix:///var/run/docker.sock collection_interval: 30s @@ -35,17 +38,14 @@ receivers: enabled: true container.network.io.usage.tx_packets: enabled: true + filelog/nginx-access-logs: include: ["${env:NGINX_ACCESS_LOG_FILE}"] operators: - # Parse the default nginx access log format. Nginx defaults to the "combined" log format - # $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" - # For more details, see https://nginx.org/en/docs/http/ngx_http_log_module.html - type: regex_parser - if: body matches '^(?P[0-9\\.]+) - (?P[^\\s]+) \\[(?P.+)\\] "(?P\\w+?) (?P.+?)" (?P[0-9]+) (?P[0-9]+) "(?P.+?)" "(?P.+?)"$' + regex: '^(?P[0-9\.]+) - (?P[^\s]+) \[(?P.+)\] "(?P\w+?) (?P.+?)" (?P[0-9]+) (?P[0-9]+) "(?P.+?)" "(?P.+?)"$' parse_from: body parse_to: attributes - regex: '^(?P[0-9\.]+) - (?P[^\s]+) \[(?P.+)\] "(?P\w+?) (?P.+?)" (?P[0-9]+) (?P[0-9]+) "(?P.+?)" "(?P.+?)"$' timestamp: parse_from: attributes.ts layout: "02/Jan/2006:15:04:05 -0700" @@ -55,13 +55,10 @@ receivers: overwrite_text: true mapping: debug: "1xx" - info: - - "2xx" - - "3xx" + info: ["2xx", "3xx"] warn: "4xx" error: "5xx" - type: remove - if: attributes.ts != nil field: attributes.ts - type: add field: attributes.source @@ -70,14 +67,10 @@ receivers: filelog/nginx-error-logs: include: ["${env:NGINX_ERROR_LOG_FILE}"] operators: - # Parse the default nginx error log format. - # YYYY/MM/DD HH:MM:SS [LEVEL] PID#TID: *CID MESSAGE - # For more details, see https://github.com/phusion/nginx/blob/master/src/core/ngx_log.c - type: regex_parser - if: body matches '^(?P.+?) \\[(?P\\w+)\\] (?P\\d+)#(?P\\d+). \\*(?P\\d+) (?P.+)$' + regex: '^(?P.+?) \[(?P\w+)\] (?P\d+)#(?P\d+). \*(?P\d+) (?P.+)$' parse_from: body parse_to: attributes - regex: '^(?P.+?) \[(?P\w+)\] (?P\d+)#(?P\d+). \*(?P\d+) (?P.+)$' timestamp: parse_from: attributes.ts layout: "2006/01/02 15:04:05" @@ -87,25 +80,19 @@ receivers: overwrite_text: true mapping: debug: "debug" - info: - - "info" - - "notice" + info: ["info", "notice"] warn: "warn" - error: - - "error" - - "crit" - - "alert" + error: ["error", "crit", "alert"] fatal: "emerg" - type: remove - if: attributes.ts != nil field: attributes.ts - type: move - if: attributes.message != nil from: attributes.message to: body - type: add field: attributes.source value: nginx + prometheus: config: global: @@ -113,35 +100,31 @@ receivers: scrape_configs: - job_name: otel-collector static_configs: - - targets: - - localhost:8888 - labels: - job_name: otel-collector + - targets: ["localhost:8888"] + labels: + job_name: otel-collector + processors: batch: send_batch_size: 10000 send_batch_max_size: 11000 timeout: 10s + resourcedetection: detectors: [env, system] - system: - hostname_sources: [os] - resourcedetection/env: - detectors: [env] timeout: 2s - override: false - resourcedetection/system: - detectors: ["system"] system: hostname_sources: ["dns", "os"] + resourcedetection/docker: detectors: [env, docker] timeout: 2s override: false + signozspanmetrics/delta: metrics_exporter: signozclickhousemetrics metrics_flush_interval: 60s - latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s ] + latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s] dimensions_cache_size: 100000 aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA enable_exp_histogram: true @@ -150,9 +133,6 @@ processors: default: default - name: deployment.environment default: default - # This is added to ensure the uniqueness of the timeseries - # Otherwise, identical timeseries produced by multiple replicas of - # collectors result in incorrect APM metrics - name: signoz.collector.id - name: service.version - name: browser.platform @@ -163,27 +143,32 @@ processors: - name: host.name - name: host.type - name: container.name + extensions: health_check: endpoint: 0.0.0.0:13133 pprof: endpoint: 0.0.0.0:1777 + exporters: clickhousetraces: datasource: tcp://clickhouse:9000/signoz_traces low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} use_new_schema: true + signozclickhousemetrics: dsn: tcp://clickhouse:9000/signoz_metrics + clickhouselogsexporter: dsn: tcp://clickhouse:9000/signoz_logs timeout: 10s use_new_schema: true - # debug: {} + otlp/nginx-logs: endpoint: "localhost:4317" tls: insecure: true + service: telemetry: logs: @@ -196,19 +181,23 @@ service: receivers: [otlp] processors: [signozspanmetrics/delta, batch] exporters: [clickhousetraces] + metrics: receivers: [otlp] processors: [batch] - exporters: [signozclickhousemetrics, resourcedetection/docker, resourcedetection/system] + exporters: [signozclickhousemetrics] + metrics/hostmetrics: receivers: [hostmetrics] - processors: [resourcedetection, resource/env] + processors: [resourcedetection] exporters: [otlp] + metrics/prometheus: receivers: [prometheus] processors: [batch] exporters: [signozclickhousemetrics] + logs: - receivers: [otlp, tcplog/docker, httplogreceiver/json] + receivers: [otlp, httplogreceiver/json, filelog/nginx-access-logs, filelog/nginx-error-logs] processors: [batch] exporters: [clickhouselogsexporter]