Bug#1025234: prometheus: flaky autopkgtest (on 32 bit archs?)

[Paul Gevers]

Source: prometheus
Version: 2.33.5+ds1-2
Severity: serious
User: debian...@lists.debian.org
Usertags: flaky

Dear maintainer(s),



Because the unstable-to-testing migration software now blocks on
regressions in testing, flaky tests, i.e. tests that flip between
passing and failing without changes to the list of installed packages,
are causing people unrelated to your package to spend time on these
tests.

Don't hesitate to reach out if you need help and some more information
from our infrastructure.

Paul

https://ci.debian.net/data/autopkgtest/testing/armhf/p/prometheus/28799146/log.gz

FAIL
FAIL    github.com/prometheus/prometheus/promql 9.743s


--- FAIL: TestQueryHistogramFromBlocksWithCompaction (11.14s)

panic: mmap, size 134217728: cannot allocate memory [recovered]
        panic: mmap, size 134217728: cannot allocate memory

goroutine 11328 [running]:
testing.tRunner.func1.2({0x88a108, 0x4c04030})
        /usr/lib/go-1.19/src/testing/testing.go:1396 +0x27c
testing.tRunner.func1()
        /usr/lib/go-1.19/src/testing/testing.go:1399 +0x3f4
panic({0x88a108, 0x4c04030})
        /usr/lib/go-1.19/src/runtime/panic.go:884 +0x23c

        
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/head_append.go:1225
 +0x88

        
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go:418
 +0x130

        
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/head_append.go:1214
 +0xa0

        
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/head_append.go:1149
 +0x24

        
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/head_append.go:1101
 +0x19c

        
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/head_append.go:983
 +0x4c
github.com/prometheus/prometheus/tsdb.(*headAppender).Commit(0x4be6140)
        
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/head_append.go:876
 +0x1410

        
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/tsdbblockutil.go:63
 +0x570

        
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/block_test.go:474
 +0x5c

        
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/db_test.go:6035
 +0x95c
github.com/prometheus/prometheus/tsdb.TestQueryHistogramFromBlocksWithCompaction.func8(0x35e6960)
        
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/db_test.go:6133
 +0x43c
testing.tRunner(0x35e6960, 0x300efb0)
        /usr/lib/go-1.19/src/testing/testing.go:1446 +0x118
created by testing.(*T).Run
        /usr/lib/go-1.19/src/testing/testing.go:1493 +0x3a0
FAIL    github.com/prometheus/prometheus/tsdb   74.037s


https://ci.debian.net/data/autopkgtest/testing/i386/p/prometheus/28799709/log.gz


fatal error: out of memory

goroutine 6356 [running]:
runtime.throw({0x89263f7, 0xd})

runtime.(*mcache).allocLarge(0xf7d13088, 0x3679508, 0x0)

runtime.mallocgc(0x3679508, 0x88c3c00, 0x1)

runtime.makeslice(0x88c3c00, 0x0, 0x6cf2a1)

runtime.makeslice64(0x88c3c00, 0x0, 0x6cf2a1)

        
/tmp/autopkgtest-lxc.cdoskdiv/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/block_test.go:671
 +0x1e0 fp=0xacf26fc sp=0xacf25d8 pc=0x871c7b0

        
/tmp/autopkgtest-lxc.cdoskdiv/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/block_test.go:607
 +0x63 fp=0xacf2730 sp=0xacf26fc pc=0x871c223
github.com/prometheus/prometheus/tsdb.TestQueryHistogramFromBlocksWithCompaction.func6(0xb7f2f00)
        
/tmp/autopkgtest-lxc.cdoskdiv/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/db_test.go:6117
 +0x15f fp=0xacf279c sp=0xacf2730 pc=0x878d6cf
testing.tRunner(0xb7f2f00, 0xc59ad5b0)

testing.(*T).Run.func1()

runtime.goexit()

created by testing.(*T).Run
        /usr/lib/go-1.19/src/testing/testing.go:1493 +0x374

goroutine 1 [chan receive]:
runtime.gopark(0x8969034, 0xac2b930, 0xe, 0x17, 0x2)

runtime.chanrecv(0xac2b900, 0xec8fc97, 0x1)

runtime.chanrecv1(0xac2b900, 0xec8fc97)

testing.(*T).Run(0xac84690, {0x89492bd, 0x2a}, 0x896842c)

testing.runTests.func1(0xac84690)

testing.tRunner(0xac84690, 0xad9fd7c)

testing.(*M).Run(0xacb3310)

        
/tmp/autopkgtest-lxc.cdoskdiv/downtmp/autopkgtest_tmp/.build/src/go.uber.org/goleak/testmain.go:53
 +0x37 fp=0xec8ff20 sp=0xec8fecc pc=0x853ebe7
github.com/prometheus/prometheus/tsdb.TestMain(0xacb3310)
        
/tmp/autopkgtest-lxc.cdoskdiv/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/db_test.go:66
 +0x205 fp=0xec8ff60 sp=0xec8ff20 pc=0x8736bf5
main.main()
        _testmain.go:487 +0x189 fp=0xec8ffc4 sp=0xec8ff60 pc=0x881c629
runtime.main()

runtime.goexit()


goroutine 2 [force gc (idle), 1 minutes]:
runtime.gopark(0x89691a8, 0x909c860, 0x11, 0x14, 0x1)

runtime.goparkunlock(...)
        /usr/lib/go-1.19/src/runtime/proc.go:369
runtime.forcegchelper()

runtime.goexit()

created by runtime.init.5
        /usr/lib/go-1.19/src/runtime/proc.go:290 +0x23

goroutine 3 [GC sweep wait]:
runtime.gopark(0x89691a8, 0x909d290, 0xc, 0x14, 0x1)

runtime.goparkunlock(...)
        /usr/lib/go-1.19/src/runtime/proc.go:369
runtime.bgsweep(0xac2a0c0)

runtime.gcenable.func1()

runtime.goexit()

created by runtime.gcenable
        /usr/lib/go-1.19/src/runtime/mgc.go:178 +0x7c

goroutine 4 [GC scavenge wait]:
runtime.gopark(0x89691a8, 0x909d640, 0xd, 0x14, 0x2)

runtime.goparkunlock(...)
        /usr/lib/go-1.19/src/runtime/proc.go:369
runtime.(*scavengerState).park(0x909d640)

runtime.bgscavenge(0xac2a0c0)

runtime.gcenable.func2()

runtime.goexit()

created by runtime.gcenable
        /usr/lib/go-1.19/src/runtime/mgc.go:179 +0xc1

goroutine 18 [finalizer wait, 1 minutes]:
runtime.gopark(0x89691a8, 0x90b1888, 0x10, 0x14, 0x1)

runtime.goparkunlock(...)
        /usr/lib/go-1.19/src/runtime/proc.go:369
runtime.runfinq()

runtime.goexit()

created by runtime.createfing
        /usr/lib/go-1.19/src/runtime/mfinal.go:157 +0x54

goroutine 6305 [chan receive]:
runtime.gopark(0x8969034, 0xc3be9470, 0xe, 0x17, 0x2)

runtime.chanrecv(0xc3be9440, 0xac3bf5b, 0x1)

runtime.chanrecv1(0xc3be9440, 0xac3bf5b)

testing.(*T).Run(0xb7f2e10, {0x8945a1a, 0x27}, 0xc59ad5b0)

github.com/prometheus/prometheus/tsdb.TestQueryHistogramFromBlocksWithCompaction(0xb7f2e10)
        
/tmp/autopkgtest-lxc.cdoskdiv/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/db_test.go:6114
 +0x151 fp=0xac3bf9c sp=0xac3bf84 pc=0x878cde1
testing.tRunner(0xb7f2e10, 0x896842c)

testing.(*T).Run.func1()

runtime.goexit()

created by testing.(*T).Run
        /usr/lib/go-1.19/src/testing/testing.go:1493 +0x374

goroutine 35 [GC worker (idle)]:
runtime.gopark(0x8969068, 0xace0690, 0x18, 0x14, 0x0)

runtime.gcBgMarkWorker()

runtime.goexit()

created by runtime.gcBgMarkStartWorkers
        /usr/lib/go-1.19/src/runtime/mgc.go:1159 +0x25

goroutine 36 [GC worker (idle)]:
runtime.gopark(0x8969068, 0xace06a8, 0x18, 0x14, 0x0)

runtime.gcBgMarkWorker()

runtime.goexit()

created by runtime.gcBgMarkStartWorkers
        /usr/lib/go-1.19/src/runtime/mgc.go:1159 +0x25
FAIL    github.com/prometheus/prometheus/tsdb   97.396s

I looked at the results of the autopkgtest of your package (because it blocked glibc). I noticed that it regularly fails. At least on 32 bits architectures, I think the test (and maybe the general use of the package?) is nearing memory limititations? Maybe the failures on 64 bit architectures were due to non-available test dependencies (I didn't check all logs). --- PASS: TestQueryHistogramFromBlocksWithCompaction/serial_blocks_with_only_histograms (1.23s) --- PASS: TestQueryHistogramFromBlocksWithCompaction/serial_blocks_with_either_histograms_or_floats_in_a_block_and_not_both (2.57s) --- PASS: TestQueryHistogramFromBlocksWithCompaction/serial_blocks_with_mix_of_histograms_and_float64 (2.59s) --- PASS: TestQueryHistogramFromBlocksWithCompaction/overlapping_blocks_with_only_histograms (1.68s) --- PASS: TestQueryHistogramFromBlocksWithCompaction/overlapping_blocks_with_only_histograms_and_only_float_in_a_series (2.63s) --- FAIL: TestQueryHistogramFromBlocksWithCompaction/overlapping_blocks_with_mix_of_histograms_and_float64 (0.44s) github.com/prometheus/prometheus/tsdb.handleChunkWriteError({0xbe5844, 0x4c04030}) github.com/prometheus/prometheus/tsdb/chunks.(*ChunkDiskMapper).WriteChunk(0x4bba000, 0x1, 0x0, 0xafc80, {0xbe9cd8, 0x4b9a6b0}, 0x93e6b8) github.com/prometheus/prometheus/tsdb.(*memSeries).mmapCurrentHeadChunk(0x4baa120, 0x4bba000) github.com/prometheus/prometheus/tsdb.(*memSeries).cutNewHeadChunk(0x4baa120, 0xdbba0, 0x1, 0x4bba000, 0x6ddd00) github.com/prometheus/prometheus/tsdb.(*memSeries).appendPreprocessor(0x4baa120, 0xdbba0, 0x1, 0x4bba000, 0x6ddd00) github.com/prometheus/prometheus/tsdb.(*memSeries).append(0x4baa120, 0xdbba0, 0x3fe1b0583e7c7c6a, 0x2, 0x4bba000, 0x6ddd00) github.com/prometheus/prometheus/tsdb.CreateBlock({0x3eea780, 0xa, 0xa}, {0x3d12000, 0x72}, 0x0, {0xbe57e0, 0x105b3f8}) github.com/prometheus/prometheus/tsdb.createBlock({0xbeb950, 0x35e6960}, {0x3d12000, 0x72}, {0x3eea780, 0xa, 0xa}) github.com/prometheus/prometheus/tsdb.TestQueryHistogramFromBlocksWithCompaction.func2(0x35e6960, {0x6142000, 0x4, 0x4}) runtime: out of memory: cannot allocate 58720256-byte block (3870720000 in use) /usr/lib/go-1.19/src/runtime/panic.go:1047 +0x4d fp=0xacf2540 sp=0xacf252c pc=0x8087f1d /usr/lib/go-1.19/src/runtime/mcache.go:235 +0x208 fp=0xacf2568 sp=0xacf2540 pc=0x805e9b8 /usr/lib/go-1.19/src/runtime/malloc.go:1029 +0x458 fp=0xacf25b0 sp=0xacf2568 pc=0x8055828 /usr/lib/go-1.19/src/runtime/slice.go:103 +0x4f fp=0xacf25c4 sp=0xacf25b0 pc=0x80a00ff /usr/lib/go-1.19/src/runtime/slice.go:117 +0x6f fp=0xacf25d8 sp=0xacf25c4 pc=0x80a01df github.com/prometheus/prometheus/tsdb.genSeriesFromSampleGenerator(0xa, 0x5, 0xea60, 0x6ddd00, 0x2bf20, 0x8968524) github.com/prometheus/prometheus/tsdb.genHistogramSeries(0xa, 0x5, 0xea60, 0x6ddd00, 0x2bf20) /usr/lib/go-1.19/src/testing/testing.go:1446 +0x113 fp=0xacf27e4 sp=0xacf279c pc=0x816b5b3 /usr/lib/go-1.19/src/testing/testing.go:1493 +0x2e fp=0xacf27f0 sp=0xacf27e4 pc=0x816c3ae /usr/lib/go-1.19/src/runtime/asm_386.s:1326 +0x1 fp=0xacf27f4 sp=0xacf27f0 pc=0x80bd3c1 /usr/lib/go-1.19/src/runtime/proc.go:363 +0x108 fp=0xec8fc18 sp=0xec8fc04 pc=0x808b228 /usr/lib/go-1.19/src/runtime/chan.go:583 +0x3f2 fp=0xec8fc60 sp=0xec8fc18 pc=0x804f3f2 /usr/lib/go-1.19/src/runtime/chan.go:442 +0x1c fp=0xec8fc74 sp=0xec8fc60 pc=0x804efcc /usr/lib/go-1.19/src/testing/testing.go:1494 +0x393 fp=0xec8fcc0 sp=0xec8fc74 pc=0x816c333 /usr/lib/go-1.19/src/testing/testing.go:1846 +0x5e fp=0xec8fce4 sp=0xec8fcc0 pc=0x816e18e /usr/lib/go-1.19/src/testing/testing.go:1446 +0x113 fp=0xec8fd2c sp=0xec8fce4 pc=0x816b5b3 testing.runTests(0xaed6000, {0x909a640, 0xc6, 0xc6}, {0xc0d9d2f513675009, 0x11765f6e595, 0x909d480}) /usr/lib/go-1.19/src/testing/testing.go:1844 +0x3a6 fp=0xec8fd90 sp=0xec8fd2c pc=0x816e076 /usr/lib/go-1.19/src/testing/testing.go:1726 +0x680 fp=0xec8fecc sp=0xec8fd90 pc=0x816cc10 go.uber.org/goleak.VerifyTestMain({0x8c0e8b8, 0xacb3310}, {0xec8ff4c, 0x2, 0x2}) /usr/lib/go-1.19/src/runtime/proc.go:250 +0x24d fp=0xec8fff0 sp=0xec8ffc4 pc=0x808ae2d /usr/lib/go-1.19/src/runtime/asm_386.s:1326 +0x1 fp=0xec8fff4 sp=0xec8fff0 pc=0x80bd3c1 /usr/lib/go-1.19/src/runtime/proc.go:363 +0x108 fp=0xac3efdc sp=0xac3efc8 pc=0x808b228 /usr/lib/go-1.19/src/runtime/proc.go:302 +0xc3 fp=0xac3eff0 sp=0xac3efdc pc=0x808b0a3 /usr/lib/go-1.19/src/runtime/asm_386.s:1326 +0x1 fp=0xac3eff4 sp=0xac3eff0 pc=0x80bd3c1 /usr/lib/go-1.19/src/runtime/proc.go:363 +0x108 fp=0xac3f7d4 sp=0xac3f7c0 pc=0x808b228 /usr/lib/go-1.19/src/runtime/mgcsweep.go:297 +0xf3 fp=0xac3f7e8 sp=0xac3f7d4 pc=0x8073743 /usr/lib/go-1.19/src/runtime/mgc.go:178 +0x27 fp=0xac3f7f0 sp=0xac3f7e8 pc=0x8064827 /usr/lib/go-1.19/src/runtime/asm_386.s:1326 +0x1 fp=0xac3f7f4 sp=0xac3f7f0 pc=0x80bd3c1 /usr/lib/go-1.19/src/runtime/proc.go:363 +0x108 fp=0xac3ffb8 sp=0xac3ffa4 pc=0x808b228 /usr/lib/go-1.19/src/runtime/mgcscavenge.go:389 +0x5f fp=0xac3ffcc sp=0xac3ffb8 pc=0x807108f /usr/lib/go-1.19/src/runtime/mgcscavenge.go:622 +0x77 fp=0xac3ffe8 sp=0xac3ffcc pc=0x80718a7 /usr/lib/go-1.19/src/runtime/mgc.go:179 +0x27 fp=0xac3fff0 sp=0xac3ffe8 pc=0x80647e7 /usr/lib/go-1.19/src/runtime/asm_386.s:1326 +0x1 fp=0xac3fff4 sp=0xac3fff0 pc=0x80bd3c1 /usr/lib/go-1.19/src/runtime/proc.go:363 +0x108 fp=0xac3e790 sp=0xac3e77c pc=0x808b228 /usr/lib/go-1.19/src/runtime/mfinal.go:180 +0xfb fp=0xac3e7f0 sp=0xac3e790 pc=0x8063a7b /usr/lib/go-1.19/src/runtime/asm_386.s:1326 +0x1 fp=0xac3e7f4 sp=0xac3e7f0 pc=0x80bd3c1 /usr/lib/go-1.19/src/runtime/proc.go:363 +0x108 fp=0xac3bedc sp=0xac3bec8 pc=0x808b228 /usr/lib/go-1.19/src/runtime/chan.go:583 +0x3f2 fp=0xac3bf24 sp=0xac3bedc pc=0x804f3f2 /usr/lib/go-1.19/src/runtime/chan.go:442 +0x1c fp=0xac3bf38 sp=0xac3bf24 pc=0x804efcc /usr/lib/go-1.19/src/testing/testing.go:1494 +0x393 fp=0xac3bf84 sp=0xac3bf38 pc=0x816c333 /usr/lib/go-1.19/src/testing/testing.go:1446 +0x113 fp=0xac3bfe4 sp=0xac3bf9c pc=0x816b5b3 /usr/lib/go-1.19/src/testing/testing.go:1493 +0x2e fp=0xac3bff0 sp=0xac3bfe4 pc=0x816c3ae /usr/lib/go-1.19/src/runtime/asm_386.s:1326 +0x1 fp=0xac3bff4 sp=0xac3bff0 pc=0x80bd3c1 /usr/lib/go-1.19/src/runtime/proc.go:363 +0x108 fp=0xac3a794 sp=0xac3a780 pc=0x808b228 /usr/lib/go-1.19/src/runtime/mgc.go:1235 +0x103 fp=0xac3a7f0 sp=0xac3a794 pc=0x8067073 /usr/lib/go-1.19/src/runtime/asm_386.s:1326 +0x1 fp=0xac3a7f4 sp=0xac3a7f0 pc=0x80bd3c1 /usr/lib/go-1.19/src/runtime/proc.go:363 +0x108 fp=0xac3af94 sp=0xac3af80 pc=0x808b228 /usr/lib/go-1.19/src/runtime/mgc.go:1235 +0x103 fp=0xac3aff0 sp=0xac3af94 pc=0x8067073 /usr/lib/go-1.19/src/runtime/asm_386.s:1326 +0x1 fp=0xac3aff4 sp=0xac3aff0 pc=0x80bd3c1

OpenPGP_signature
Description: OpenPGP digital signature