Any idea how to fix it? ts=2023-10-13T08:38:14.786Z caller=main.go:583 level=info msg="Starting Prometheus Server" mode=server version="(version=2.47.1, branch=HEAD, revision=c4d1a8beff37cc004f1dc4ab9d2e73193f51aaeb)" ts=2023-10-13T08:38:14.786Z caller=main.go:588 level=info build_context="(go=go1.21.1, platform=linux/amd64, user=root@4829330363be, date=20231004-10:31:16, tags=netgo,builtinassets,stringlabels)" ts=2023-10-13T08:38:14.786Z caller=main.go:589 level=info host_details="(Linux 5.15.0-1041-azure #48-Ubuntu SMP Tue Jun 20 20:34:08 UTC 2023 x86_64 prometheus-5765644cf4-dczhv (none))" ts=2023-10-13T08:38:14.786Z caller=main.go:590 level=info fd_limits="(soft=1048576, hard=1048576)" ts=2023-10-13T08:38:14.786Z caller=main.go:591 level=info vm_limits="(soft=unlimited, hard=unlimited)" ts=2023-10-13T08:38:14.825Z caller=web.go:566 level=info component=web msg="Start listening for connections" address=0.0.0.0:9290 ts=2023-10-13T08:38:14.826Z caller=main.go:1024 level=info msg="Starting TSDB ..." ts=2023-10-13T08:38:14.827Z caller=tls_config.go:274 level=info component=web msg="Listening on" address=[::]:9290 ts=2023-10-13T08:38:14.827Z caller=tls_config.go:310 level=info component=web msg="TLS is enabled." http2=true address=[::]:9290
ts=2023-10-13T08:38:14.940Z caller=db.go:930 level=info component=tsdb msg="Found and deleted tmp block dir" dir=data/01HCM2ZA20JWZ3SSB4MCY7KWJB.tmp-for-creation ts=2023-10-13T08:38:14.945Z caller=dir_locker.go:77 level=warn component=tsdb msg="A lockfile from a previous execution already existed. It was replaced" file=/prometheus/data/lock ts=2023-10-13T08:38:14.985Z caller=head.go:600 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" ts=2023-10-13T08:38:14.986Z caller=head.go:666 level=error component=tsdb msg="Loading on-disk chunks failed" err="iterate on on-disk chunks: corruption in head chunk file data/chunks_head/000001: head chunk file has some unread data, but doesn't include enough bytes to read the chunk header - required:131078, available:131072, file:1" ts=2023-10-13T08:38:14.986Z caller=head.go:906 level=info component=tsdb msg="Deleting mmapped chunk files" ts=2023-10-13T08:38:14.988Z caller=head.go:912 level=info component=tsdb msg="Deleting mmapped chunk files" ts=2023-10-13T08:38:15.000Z caller=head.go:922 level=info component=tsdb msg="Deletion of mmap chunk files successful, reattempting m-mapping the on-disk chunks" ts=2023-10-13T08:38:15.000Z caller=head.go:681 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=14.995821ms ts=2023-10-13T08:38:15.000Z caller=head.go:689 level=info component=tsdb msg="Replaying WAL, this may take a while" ts=2023-10-13T08:38:15.251Z caller=db.go:898 level=warn component=tsdb msg="Encountered WAL read error, attempting repair" err="read records: corruption in segment data/wal/00000000 at 1212416: unexpected non-zero byte in padded page" ts=2023-10-13T08:38:15.252Z caller=wlog.go:426 level=warn component=tsdb msg="Starting corruption repair" segment=0 offset=1212416 ts=2023-10-13T08:38:15.259Z caller=wlog.go:434 level=warn component=tsdb msg="Deleting all segments newer than corrupted segment" segment=0 ts=2023-10-13T08:38:15.279Z caller=wlog.go:456 level=warn component=tsdb msg="Rewrite corrupted segment" segment=0 ts=2023-10-13T08:38:23.386Z caller=db.go:902 level=info component=tsdb msg="Successfully repaired WAL" ts=2023-10-13T08:38:23.391Z caller=main.go:1045 level=info fs_type=fe534d42 ts=2023-10-13T08:38:23.391Z caller=main.go:1048 level=info msg="TSDB started" ts=2023-10-13T08:38:43.281Z caller=compact.go:562 level=error component=tsdb msg="removed tmp folder after failed compaction" err="unlinkat data/01HCM306H4R5JVAP85K42A11C7.tmp-for-creation: directory not empty" ts=2023-10-13T08:38:43.281Z caller=db.go:984 level=error component=tsdb msg="compaction failed" err="compact head: persist head block: 2 errors: populate block: add series: read symbols: invalid checksum; read symbols: invalid checksum" ts=2023-10-13T08:38:44.734Z caller=compact.go:562 level=error component=tsdb msg="removed tmp folder after failed compaction" err="unlinkat data/01HCM307SSQPBMGTYZ4VK7D117.tmp-for-creation: directory not empty" ts=2023-10-13T08:38:44.734Z caller=db.go:984 level=error component=tsdb msg="compaction failed" err="compact head: persist head block: 2 errors: populate block: add series: read symbols: invalid checksum; read symbols: invalid checksum" ts=2023-10-13T08:38:47.040Z caller=compact.go:562 level=error component=tsdb msg="removed tmp folder after failed compaction" err="unlinkat data/01HCM30A6F8GEXNYT4HJC8TN88.tmp-for-creation: directory not empty" ts=2023-10-13T08:38:47.040Z caller=db.go:984 level=error component=tsdb msg="compaction failed" err="compact head: persist head block: 2 errors: populate block: add series: read symbols: invalid checksum; read symbols: invalid checksum" ts=2023-10-13T08:38:51.483Z caller=compact.go:562 level=error component=tsdb msg="removed tmp folder after failed compaction" err="unlinkat data/01HCM30ED1P7AKQJ7M6A9D5GR2.tmp-for-creation: directory not empty" ts=2023-10-13T08:38:51.483Z caller=db.go:984 level=error component=tsdb msg="compaction failed" err="compact head: persist head block: 2 errors: populate block: add series: read symbols: invalid checksum; read symbols: invalid checksum" ts=2023-10-13T08:38:59.826Z caller=compact.go:562 level=error component=tsdb msg="removed tmp folder after failed compaction" err="unlinkat data/01HCM30PMVZ1AJJ03HQYYNX6RJ.tmp-for-creation: directory not empty" ts=2023-10-13T08:38:59.827Z caller=db.go:984 level=error component=tsdb msg="compaction failed" err="compact head: persist head block: 2 errors: populate block: add series: read symbols: invalid checksum; read symbols: invalid checksum" ts=2023-10-13T08:39:13.910Z caller=stdlib.go:105 level=error component=web caller="http: TLS handshake error from 10.104.60.63:46062" msg=EOF # kubectl get pods -n grafana NAME READY STATUS RESTARTS AGE grafana-5d5c88677b-wvkhm 1/1 Running 0 75m influxdb-df6b77f54-hzzs9 1/1 Running 0 29d prometheus-5765644cf4-dczhv 1/1 Running 1 (11m ago) 166m $ ls -tlr total 20 -rwxrwxrwx 1 root root 0 Oct 13 14:02 lock drwxrwxrwx 2 root root 0 Oct 13 14:02 wal drwxrwxrwx 2 root root 0 Oct 13 14:02 chunks_head -rwxrwxrwx 1 root root 20001 Oct 13 16:38 queries.active drwxrwxrwx 2 root root 0 Oct 13 16:38 01HCM306H4R5JVAP85K42A11C7.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:38 01HCM307SSQPBMGTYZ4VK7D117.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:38 01HCM30A6F8GEXNYT4HJC8TN88.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:38 01HCM30ED1P7AKQJ7M6A9D5GR2.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:38 01HCM30PMVZ1AJJ03HQYYNX6RJ.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:39 01HCM316KKGX4ZC2SJV2DPD4YQ.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:39 01HCM3266SHYDKQEYR9BH3K2NK.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:40 01HCM3415EK8FCQ5DW3SFVE6HE.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:41 01HCM35W4GP5R0GBDYSMQM3VP0.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:42 01HCM37Q2W4SY54V7Y398718TJ.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:43 01HCM39J1BW8QEGV3W2M89YGSK.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:44 01HCM3BCZEJ9B56XYJZJTME4PK.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:45 01HCM3D7Y3E7AY4Z9XF6HT4WVZ.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:46 01HCM3F2YB9B8C6DGV7T9VXGED.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:47 01HCM3GXZ1F13VVZQKCHM696EA.tmp-for-creation -- You received this message because you are subscribed to the Google Groups "Prometheus Users" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. To view this discussion on the web visit https://groups.google.com/d/msgid/prometheus-users/b556b7ff-a51e-4591-a717-4d5e2fcb8991n%40googlegroups.com.
