provisioned-o11y-alloy-1 alloy ts=2025-09-12T11:25:06.632943746Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.provisioned_logs component=client host=logs.example.org status=400 tenant=provisioned-0dbdbdc402d24ac894e2bec924696531 error="server returned HTTP status 400 Bad Request (400): entry for stream '{__address__=\"192.168.144.155:10250\", __meta_kubernetes_node_address_Hostname=\"6191a3ad-f0fa-456f-9217-3020989c3a6d-59777554db-wmzsw\", __meta_kubernetes_node_address_InternalIP=\"192.168.144.155\", __meta_kubernetes_node_annotation_alpha_kubernetes_io_provided_node_ip=\"192.168.144.155\", __meta_kubernetes_node_annotation_cluster_k8s_io_machine=\"6191a3ad-f0fa-456f-9217-3020989c3a6d/6191a3ad-f0fa-456f-9217-3020989c3a6d-59777554db-wmzsw\", __meta_kubernetes_node_annotation_csi_volume_kubernetes_io_nodeid=\"{\\\"cinder.csi.openstack.org\\\":\\\"655d466a-ad4b-4b13-bcbc-c2bb42484fb7\\\"}\", __meta_kubernetes_node_annotation_network_cilium_io_ipv4_cilium_host=\"172.28.3.181\", __meta_kubernetes_node_annotation_network_cilium_io_ipv4_health_ip=\"172.28.3.161\", __meta_kubernetes_node_annotation_network_cilium_io_ipv4_pod_cidr=\"172.28.3.0/24\", __meta_kubernetes_node_annotation_node_alpha_kubernetes_io_ttl=\"0\", __meta_kubernetes_node_annotation_volumes_kubernetes_io_controller_managed_attach_detach=\"true\", __meta_kuber"

loki.write "provisioned_logs" {
  endpoint {
    url = "http://loki:3100/loki/api/v1/push"
  }
}

loki.source.file "dpkg_log" {
  targets    = [
    {__path__ = "/var/log/dpkg.log"},
  ]

  forward_to = [loki.relabel.generate_labels.receiver]
}

loki.relabel "generate_labels" {
  forward_to = [loki.write.provisioned_logs.receiver]

  rule {
    target_label = "__foo1"
    replacement  = "bar"
  }
  rule {
    target_label = "__foo2"
    replacement  = "bar"
  }
  rule {
    target_label = "__foo3"
    replacement  = "bar"
  }
  rule {
    target_label = "__foo4"
    replacement  = "bar"
  }
  rule {
    target_label = "__foo5"
    replacement  = "bar"
  }
  rule {
    target_label = "__foo6"
    replacement  = "bar"
  }
  rule {
    target_label = "__foo7"
    replacement  = "bar"
  }
  rule {
    target_label = "__foo8"
    replacement  = "bar"
  }
  rule {
    target_label = "__foo9"
    replacement  = "bar"
  }
  rule {
    target_label = "__foo10"
    replacement  = "bar"
  }
  rule {
    target_label = "__foo11"
    replacement  = "bar"
  }
  rule {
    target_label = "__foo12"
    replacement  = "bar"
  }
  rule {
    target_label = "__foo13"
    replacement  = "bar"
  }
  rule {
    target_label = "__foo14"
    replacement  = "bar"
  }
  rule {
    target_label = "__foo15"
    replacement  = "bar"
  }
}

loki  | level=info ts=2025-09-12T12:57:27.484485013Z caller=main.go:126 msg="Starting Loki" version="(version=3.5.5, branch=release-3.5.x, revision=5aa8bd27)"
loki  | level=info ts=2025-09-12T12:57:27.484516346Z caller=main.go:127 msg="Loading configuration file" filename=/etc/loki/local-config.yaml
loki  | level=info ts=2025-09-12T12:57:27.488224471Z caller=server.go:368 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
loki  | level=info ts=2025-09-12T12:57:27.49158218Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
loki  | level=info ts=2025-09-12T12:57:27.491634055Z caller=table_manager.go:300 index-store=tsdb-2020-10-24 msg="query readiness setup completed" duration=1.333µs distinct_users_len=0 distinct_users=
loki  | level=info ts=2025-09-12T12:57:27.491672263Z caller=shipper.go:165 index-store=tsdb-2020-10-24 msg="starting index shipper in RW mode"
loki  | level=info ts=2025-09-12T12:57:27.491939555Z caller=head_manager.go:311 index-store=tsdb-2020-10-24 component=tsdb-head-manager msg="loaded wals by period" groups=0
loki  | level=info ts=2025-09-12T12:57:27.491980972Z caller=manager.go:86 index-store=tsdb-2020-10-24 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
loki  | level=info ts=2025-09-12T12:57:27.491989555Z caller=head_manager.go:311 index-store=tsdb-2020-10-24 component=tsdb-head-manager msg="loaded wals by period" groups=0
loki  | level=info ts=2025-09-12T12:57:27.49498193Z caller=mapper.go:47 msg="cleaning up mapped rules directory" path=/loki/rules-temp
loki  | level=info ts=2025-09-12T12:57:27.495657222Z caller=worker.go:131 component=querier msg="Starting querier worker using query-scheduler and scheduler ring for addresses"
loki  | level=info ts=2025-09-12T12:57:27.509311097Z caller=module_service.go:82 msg=starting module=server
loki  | level=info ts=2025-09-12T12:57:27.509347555Z caller=module_service.go:82 msg=starting module=cache-generation-loader
loki  | level=info ts=2025-09-12T12:57:27.509366097Z caller=module_service.go:82 msg=starting module=query-frontend-tripperware
loki  | level=info ts=2025-09-12T12:57:27.509368472Z caller=module_service.go:82 msg=starting module=memberlist-kv
loki  | level=info ts=2025-09-12T12:57:27.509389555Z caller=module_service.go:82 msg=starting module=query-scheduler-ring
loki  | level=info ts=2025-09-12T12:57:27.509404472Z caller=module_service.go:82 msg=starting module=store
loki  | level=info ts=2025-09-12T12:57:27.509419972Z caller=module_service.go:82 msg=starting module=ring
loki  | level=info ts=2025-09-12T12:57:27.509571388Z caller=ring.go:361 msg="ring doesn't exist in KV store yet"
loki  | level=info ts=2025-09-12T12:57:27.50957168Z caller=basic_lifecycler.go:301 msg="instance not found in the ring" instance=b3bb0f6bdd4e ring=scheduler
loki  | level=info ts=2025-09-12T12:57:27.509612388Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
loki  | level=info ts=2025-09-12T12:57:27.509653513Z caller=ring.go:361 msg="ring doesn't exist in KV store yet"
loki  | level=info ts=2025-09-12T12:57:27.509684805Z caller=module_service.go:82 msg=starting module=analytics
loki  | level=info ts=2025-09-12T12:57:27.50970418Z caller=module_service.go:82 msg=starting module=ingester-querier
loki  | level=info ts=2025-09-12T12:57:27.509716055Z caller=ringmanager.go:186 msg="waiting until scheduler is JOINING in the ring"
loki  | level=info ts=2025-09-12T12:57:27.510810472Z caller=module_service.go:82 msg=starting module=compactor
loki  | level=info ts=2025-09-12T12:57:27.510877597Z caller=ring.go:361 msg="ring doesn't exist in KV store yet"
loki  | level=info ts=2025-09-12T12:57:27.510915263Z caller=module_service.go:82 msg=starting module=ingester
loki  | level=info ts=2025-09-12T12:57:27.510916972Z caller=module_service.go:82 msg=starting module=rule-evaluator
loki  | level=info ts=2025-09-12T12:57:27.510954805Z caller=module_service.go:82 msg=starting module=distributor
loki  | level=info ts=2025-09-12T12:57:27.511000305Z caller=ingester.go:564 component=ingester msg="recovering from checkpoint"
loki  | level=info ts=2025-09-12T12:57:27.511137597Z caller=recovery.go:41 component=ingester msg="no checkpoint found, treating as no-op"
loki  | level=error ts=2025-09-12T12:57:27.511121638Z caller=ratestore.go:109 msg="error getting ingester clients" err="empty ring"
loki  | level=info ts=2025-09-12T12:57:27.511180347Z caller=module_service.go:82 msg=starting module=ruler
loki  | level=info ts=2025-09-12T12:57:27.511229888Z caller=ruler.go:533 msg="ruler up and running"
loki  | level=info ts=2025-09-12T12:57:27.51127543Z caller=basic_lifecycler.go:301 msg="instance not found in the ring" instance=b3bb0f6bdd4e ring=compactor
loki  | level=info ts=2025-09-12T12:57:27.51128293Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
loki  | level=info ts=2025-09-12T12:57:27.51131468Z caller=ring.go:361 component=distributor msg="ring doesn't exist in KV store yet"
loki  | level=info ts=2025-09-12T12:57:27.511338722Z caller=compactor.go:443 msg="waiting until compactor is JOINING in the ring"
loki  | level=info ts=2025-09-12T12:57:27.511389972Z caller=basic_lifecycler.go:301 component=distributor msg="instance not found in the ring" instance=b3bb0f6bdd4e ring=distributor
loki  | level=info ts=2025-09-12T12:57:27.511369972Z caller=ingester.go:580 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=368.792µs errors=false
loki  | level=info ts=2025-09-12T12:57:27.51149993Z caller=ingester.go:586 component=ingester msg="recovering from WAL"
loki  | level=info ts=2025-09-12T12:57:27.511848097Z caller=ingester.go:602 component=ingester msg="WAL segment recovery finished" elapsed=852.125µs errors=false
loki  | level=info ts=2025-09-12T12:57:27.511855305Z caller=ingester.go:550 component=ingester msg="closing recoverer"
loki  | level=info ts=2025-09-12T12:57:27.511874388Z caller=ingester.go:558 component=ingester msg="WAL recovery finished" time=878.417µs
loki  | level=info ts=2025-09-12T12:57:27.511955805Z caller=lifecycler.go:677 component=ingester msg="not loading tokens from file, tokens file path is empty"
loki  | level=info ts=2025-09-12T12:57:27.511989722Z caller=lifecycler.go:704 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
loki  | level=info ts=2025-09-12T12:57:27.51201918Z caller=lifecycler.go:546 component=ingester msg="auto-joining cluster after timeout" ring=ingester
loki  | level=info ts=2025-09-12T12:57:27.512066805Z caller=wal.go:157 msg=started component=wal
loki  | level=info ts=2025-09-12T12:57:27.512108347Z caller=ingester.go:771 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=10.331352252s
loki  | level=info ts=2025-09-12T12:57:27.627439805Z caller=compactor.go:447 msg="compactor is JOINING in the ring"
loki  | level=info ts=2025-09-12T12:57:27.627556138Z caller=compactor.go:457 msg="waiting until compactor is ACTIVE in the ring"
loki  | level=info ts=2025-09-12T12:57:27.641895138Z caller=ringmanager.go:190 msg="scheduler is JOINING in the ring"
loki  | level=info ts=2025-09-12T12:57:27.642003138Z caller=ringmanager.go:199 msg="waiting until scheduler is ACTIVE in the ring"
loki  | level=info ts=2025-09-12T12:57:27.777747347Z caller=compactor.go:461 msg="compactor is ACTIVE in the ring"
loki  | level=info ts=2025-09-12T12:57:27.789692138Z caller=ringmanager.go:203 msg="scheduler is ACTIVE in the ring"
loki  | level=info ts=2025-09-12T12:57:27.789795013Z caller=module_service.go:82 msg=starting module=query-scheduler
loki  | level=info ts=2025-09-12T12:57:27.789904097Z caller=module_service.go:82 msg=starting module=query-frontend
loki  | level=info ts=2025-09-12T12:57:27.789923888Z caller=module_service.go:82 msg=starting module=querier
loki  | level=info ts=2025-09-12T12:57:27.789980222Z caller=loki.go:581 msg="Loki started" startup_time=359.314167ms
loki  | level=info ts=2025-09-12T12:57:30.791217417Z caller=scheduler.go:653 msg="this scheduler is in the ReplicationSet, will now accept requests."
loki  | level=info ts=2025-09-12T12:57:30.791401125Z caller=worker.go:232 component=querier msg="adding connection" addr=127.0.0.1:9095
loki  | level=info ts=2025-09-12T12:57:32.780061251Z caller=compactor.go:522 msg="this instance has been chosen to run the compactor, starting compactor"
loki  | level=info ts=2025-09-12T12:57:32.780198668Z caller=compactor.go:551 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
loki  | level=info ts=2025-09-12T12:57:37.791827087Z caller=frontend_scheduler_worker.go:107 msg="adding connection to scheduler" addr=127.0.0.1:9095
alloy-1  | ts=2025-09-12T12:57:48.156563467Z level=info "boringcrypto enabled"=false
alloy-1  | ts=2025-09-12T12:57:48.151511009Z level=info source=/go/pkg/mod/github.com/!kim!machine!gun/automemlimit@v0.7.1/memlimit/memlimit.go:175 msg="memory is not limited, skipping" package=github.com/KimMachineGun/automemlimit/memlimit
alloy-1  | ts=2025-09-12T12:57:48.156613675Z level=info msg="no peer discovery configured: both join and discover peers are empty" service=cluster
alloy-1  | ts=2025-09-12T12:57:48.156616967Z level=info msg="running usage stats reporter"
alloy-1  | ts=2025-09-12T12:57:48.15661905Z level=info msg="starting complete graph evaluation" controller_path=/ controller_id="" trace_id=29cc08169dffa0e028ecb57c19b0c759
alloy-1  | ts=2025-09-12T12:57:48.156632384Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=29cc08169dffa0e028ecb57c19b0c759 node_id=otel duration=1µs
alloy-1  | ts=2025-09-12T12:57:48.156635759Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=29cc08169dffa0e028ecb57c19b0c759 node_id=livedebugging duration=22.792µs
alloy-1  | ts=2025-09-12T12:57:48.156638009Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=29cc08169dffa0e028ecb57c19b0c759 node_id=loki.write.provisioned_logs duration=683.042µs
alloy-1  | ts=2025-09-12T12:57:48.1566403Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=29cc08169dffa0e028ecb57c19b0c759 node_id=loki.relabel.generate_labels duration=156.625µs
alloy-1  | ts=2025-09-12T12:57:48.156642467Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=29cc08169dffa0e028ecb57c19b0c759 node_id=loki.source.file.dpkg_log duration=169.792µs
alloy-1  | ts=2025-09-12T12:57:48.156645092Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=29cc08169dffa0e028ecb57c19b0c759 node_id=tracing duration=4.125µs
alloy-1  | ts=2025-09-12T12:57:48.1566473Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=29cc08169dffa0e028ecb57c19b0c759 node_id=labelstore duration=3.083µs
alloy-1  | ts=2025-09-12T12:57:48.156650134Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=29cc08169dffa0e028ecb57c19b0c759 node_id=logging duration=96.833µs
alloy-1  | ts=2025-09-12T12:57:48.156672884Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=29cc08169dffa0e028ecb57c19b0c759 node_id=remotecfg duration=19µs
alloy-1  | ts=2025-09-12T12:57:48.156687425Z level=info msg="applying non-TLS config to HTTP server" service=http
alloy-1  | ts=2025-09-12T12:57:48.156692842Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=29cc08169dffa0e028ecb57c19b0c759 node_id=http duration=10.625µs
alloy-1  | ts=2025-09-12T12:57:48.156702967Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=29cc08169dffa0e028ecb57c19b0c759 node_id=ui duration=2.292µs
alloy-1  | ts=2025-09-12T12:57:48.156728175Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=29cc08169dffa0e028ecb57c19b0c759 node_id=cluster duration=1.5µs
alloy-1  | ts=2025-09-12T12:57:48.156731717Z level=info msg="finished complete graph evaluation" controller_path=/ controller_id="" trace_id=29cc08169dffa0e028ecb57c19b0c759 duration=1.352583ms
alloy-1  | ts=2025-09-12T12:57:48.156961634Z level=info msg="scheduling loaded components and services"
alloy-1  | ts=2025-09-12T12:57:48.157207134Z level=info msg="starting cluster node" service=cluster peers_count=0 peers="" advertise_addr=127.0.0.1:12345 minimum_cluster_size=0 minimum_size_wait_timeout=0s
alloy-1  | ts=2025-09-12T12:57:48.157646175Z level=info msg="tail routine: started" component_path=/ component_id=loki.source.file.dpkg_log component=tailer path=/var/log/dpkg.log
alloy-1  | ts=2025-09-12T12:57:48.1576568Z level=info msg="peers changed" service=cluster peers_count=1 min_cluster_size=0 peers=6174ef1e8f05
alloy-1  | ts=2025-09-12T12:57:48.157773759Z level=info msg="Seeked /var/log/dpkg.log - &{Offset:0 Whence:0}" component_path=/ component_id=loki.source.file.dpkg_log component=tailer
alloy-1  | ts=2025-09-12T12:57:48.159895134Z level=info msg="now listening for http traffic" service=http addr=0.0.0.0:12345
alloy-1  | ts=2025-09-12T12:57:49.267348676Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.provisioned_logs component=client host=loki:3100 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): entry for stream '{__foo1=\"bar\", __foo10=\"bar\", __foo11=\"bar\", __foo12=\"bar\", __foo13=\"bar\", __foo14=\"bar\", __foo15=\"bar\", __foo2=\"bar\", __foo3=\"bar\", __foo4=\"bar\", __foo5=\"bar\", __foo6=\"bar\", __foo7=\"bar\", __foo8=\"bar\", __foo9=\"bar\", filename=\"/var/log/dpkg.log\", service_name=\"unknown_service\"}' has 16 label names; limit 15"
loki     | level=error ts=2025-09-12T12:57:49.266893426Z caller=manager.go:50 component=distributor path=write msg="write operation failed" details="entry for stream '{__foo1=\"bar\", __foo10=\"bar\", __foo11=\"bar\", __foo12=\"bar\", __foo13=\"bar\", __foo14=\"bar\", __foo15=\"bar\", __foo2=\"bar\", __foo3=\"bar\", __foo4=\"bar\", __foo5=\"bar\", __foo6=\"bar\", __foo7=\"bar\", __foo8=\"bar\", __foo9=\"bar\", filename=\"/var/log/dpkg.log\", service_name=\"unknown_service\"}' has 16 label names; limit 15" org_id=fake
loki     | level=info ts=2025-09-12T12:57:57.514158805Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
loki     | level=info ts=2025-09-12T12:57:57.514273805Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
loki     | level=info ts=2025-09-12T12:57:57.51429168Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"