It would appear you have a slow memory leak. I would examine the process memory usage every 30 minutes to see if it is growing... then find your memory leak using a heap examination tool.
Sent from my iPhone > On Nov 4, 2018, at 11:22 AM, Aditya Giri <g.adi1...@gmail.com> wrote: > > fatal error: runtime: out of memory > message='' > runtime stack: > runtime.throw(0x19e73ac, 0x16) > golang/go/src/runtime/panic.go:605 +0x95 fp=0x7f47a4aa5c10 > sp=0x7f47a4aa5bf0 pc=0x42d815 > runtime.sysMap(0xc78d0a0000, 0x590000, 0x60000d3400, 0x27e2538) > golang/go/src/runtime/mem_linux.go:216 +0x1d0 fp=0x7f47a4aa5c50 > sp=0x7f47a4aa5c10 pc=0x417880 > runtime.(*mheap).sysAlloc(0x27c7fc0, 0x590000, 0x0) > golang/go/src/runtime/malloc.go:470 +0xd7 fp=0x7f47a4aa5ca0 > sp=0x7f47a4aa5c50 pc=0x412007 > runtime.(*mheap).grow(0x27c7fc0, 0x2c6, 0x0) > golang/go/src/runtime/mheap.go:887 +0x60 fp=0x7f47a4aa5cf8 > sp=0x7f47a4aa5ca0 pc=0x424a20 > runtime.(*mheap).allocSpanLocked(0x27c7fc0, 0x2c6, 0x27e2548, 0x0) > golang/go/src/runtime/mheap.go:800 +0x334 fp=0x7f47a4aa5d38 > sp=0x7f47a4aa5cf8 pc=0x424884 > runtime.(*mheap).alloc_m(0x27c7fc0, 0x2c6, 0x100, 0x0) > golang/go/src/runtime/mheap.go:666 +0x118 fp=0x7f47a4aa5d78 > sp=0x7f47a4aa5d38 pc=0x4240f8 > runtime.(*mheap).alloc.func1() > golang/go/src/runtime/mheap.go:733 +0x4d fp=0x7f47a4aa5db0 > sp=0x7f47a4aa5d78 pc=0x45a01d > runtime.systemstack(0x7f47a4aa5dd0) > golang/go/src/runtime/asm_amd64.s:360 +0xab fp=0x7f47a4aa5db8 > sp=0x7f47a4aa5db0 pc=0x45c1db > runtime.(*mheap).alloc(0x27c7fc0, 0x2c6, 0x7f477e010100, 0x7f47a4aa5e50) > golang/go/src/runtime/mheap.go:732 +0xa1 fp=0x7f47a4aa5e08 > sp=0x7f47a4aa5db8 pc=0x424411 > runtime.largeAlloc(0x58c000, 0x450001, 0xc78cba6000) > golang/go/src/runtime/malloc.go:827 +0x98 fp=0x7f47a4aa5e50 > sp=0x7f47a4aa5e08 pc=0x412e08 > runtime.mallocgc.func1() > golang/go/src/runtime/malloc.go:722 +0x46 fp=0x7f47a4aa5e80 > sp=0x7f47a4aa5e50 pc=0x458f96 > runtime.systemstack(0xc42001e000) > golang/go/src/runtime/asm_amd64.s:344 +0x79 fp=0x7f47a4aa5e88 > sp=0x7f47a4aa5e80 pc=0x45c1a9 > runtime.mstart() > golang/go/src/runtime/proc.go:1135 fp=0x7f47a4aa5e90 sp=0x7f47a4aa5e88 > pc=0x432010 > message='' > goroutine 271632910 [running]: > runtime.systemstack_switch() > golang/go/src/runtime/asm_amd64.s:298 fp=0xc4cfba1bd0 sp=0xc4cfba1bc8 > pc=0x45c120 > runtime.mallocgc(0x58c000, 0x17db380, 0x38c001, 0xc78cba6000) > golang/go/src/runtime/malloc.go:721 +0x7ae fp=0xc4cfba1c78 > sp=0xc4cfba1bd0 pc=0x412c5e > runtime.growslice(0x17db380, 0xc78cba6000, 0x8e000, 0x8e000, 0x8e001, > 0xc78cba6000, 0x71800, 0x8e000) > golang/go/src/runtime/slice.go:147 +0x139 fp=0xc4cfba1ce0 sp=0xc4cfba1c78 > pc=0x445b49 > >> On Sun, Nov 4, 2018 at 10:46 PM Robert Engels <reng...@ix.netcom.com> wrote: >> Where is the panic listing? >> >> Sent from my iPhone >> >>> On Nov 4, 2018, at 2:28 AM, Aditya Giri <g.adi1...@gmail.com> wrote: >>> >>> I have 3 crons running in my application in production. 1st cron fetched >>> around 10k data and started to process. While the second cron tried to >>> fetch data and process it, IO memory issue occurred. Runtime panic. >>> >>> goroutine 250054613 [select, 129 minutes]: >>> runtime.gopark(0x1a3efa0, 0x0, 0x19cf5b8, 0x6, 0x18, 0x1) >>> golang/go/src/runtime/proc.go:287 +0x12c fp=0xc4226acbc0 sp=0xc4226acb90 >>> pc=0x42f3bc >>> runtime.selectgo(0xc4226acf10, 0xc4d08518c0) >>> golang/go/src/runtime/select.go:395 +0x1149 fp=0xc4226ace38 sp=0xc4226acbc0 >>> pc=0x43f869 >>> >>> >>> >>> goroutine 248386142 [IO wait, 63 minutes]: >>> runtime.gopark(0x1a3ef38, 0x7f47a61751b0, 0x19cfe1f, 0x7, 0xc42001e01b, 0x5) >>> golang/go/src/runtime/proc.go:287 +0x12c fp=0xc420ebfa88 sp=0xc420ebfa58 >>> pc=0x42f3bc >>> runtime.netpollblock(0x7f47a6175188, 0x72, 0x8000) >>> golang/go/src/runtime/netpoll.go:366 +0xaa fp=0xc420ebfad0 sp=0xc420ebfa88 >>> pc=0x42a4ca >>> internal/poll.runtime_pollWait(0x7f47a6175188, 0x72, 0x0) >>> golang/go/src/runtime/netpoll.go:173 +0x57 fp=0xc420ebfaf8 sp=0xc420ebfad0 >>> pc=0x429ac7 >>> internal/poll.(*pollDesc).wait(0xc4202a9f98, 0x72, 0xffffffffffffff00, >>> 0x25ea780, 0x25dea08) >>> golang/go/src/internal/poll/fd_poll_runtime.go:85 +0xae fp=0xc420ebfb30 >>> sp=0xc420ebfaf8 pc=0x49b87e >>> internal/poll.(*pollDesc).waitRead(0xc4202a9f98, 0xc4fcab4000, 0x8000, >>> 0x8000) >>> golang/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d fp=0xc420ebfb68 >>> sp=0xc420ebfb30 pc=0x49b8fd >>> internal/poll.(*FD).Read(0xc4202a9f80, 0xc4fcab4000, 0x8000, 0x8000, 0x0, >>> 0x0, 0x0) >>> golang/go/src/internal/poll/fd_unix.go:126 +0x18a fp=0xc420ebfbb8 >>> sp=0xc420ebfb68 pc=0x49c69a >>> net.(*netFD).Read(0xc4202a9f80, 0xc4fcab4000, 0x8000, 0x8000, 0x11, 0x0, >>> 0x0) >>> golang/go/src/net/fd_unix.go:202 +0x52 fp=0xc420ebfc20 sp=0xc420ebfbb8 >>> pc=0x505cc2 >>> net.(*conn).Read(0xc4212b0180, 0xc4fcab4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) >>> golang/go/src/net/net.go:176 +0x6d fp=0xc420ebfc88 sp=0xc420ebfc20 >>> pc=0x518aad >>> bufio.(*Reader).Read(0xc568be77a0, 0xc4210ce038, 0x9, 0x9, 0x9, 0x0, 0x0) >>> golang/go/src/bufio/bufio.go:213 +0x30b fp=0xc420ebfd20 sp=0xc420ebfc88 >>> pc=0x59620b >>> io.ReadAtLeast(0x25e0640, 0xc568be77a0, 0xc4210ce038, 0x9, 0x9, 0x9, >>> 0xc42026d080, 0x1, 0x0) >>> golang/go/src/io/io.go:309 +0x86 fp=0xc420ebfd80 sp=0xc420ebfd20 >>> pc=0x478276 >>> io.ReadFull(0x25e0640, 0xc568be77a0, 0xc4210ce038, 0x9, 0x9, 0x6b15804, >>> 0x6b15804fcd8af2c, 0x5bdca537) >>> golang/go/src/io/io.go:327 +0x58 fp=0xc420ebfdd8 sp=0xc420ebfd80 >>> pc=0x4783e8 >>> >>> Can someone please shed some light on this behaviour. >>> -- >>> You received this message because you are subscribed to the Google Groups >>> "golang-nuts" group. >>> To unsubscribe from this group and stop receiving emails from it, send an >>> email to golang-nuts+unsubscr...@googlegroups.com. >>> For more options, visit https://groups.google.com/d/optout. -- You received this message because you are subscribed to the Google Groups "golang-nuts" group. To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscr...@googlegroups.com. For more options, visit https://groups.google.com/d/optout.
