Possible memory leak, with reproduce repo

[wxsms]

Possible memory leak, with reproduce repo

[mattn]

This is memory usage of small program below for 7 minutes.

package main

import (
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
)

func main() {
	f := func() {
		db, err := sql.Open("sqlite3", "file::memory:")
		if err != nil {
			panic(err)
		}
		defer db.Close()

		if _, err := db.Exec(`CREATE TABLE dummy_table (id INT PRIMARY KEY,f1 VARCHAR(50),f2 VARCHAR(50),f3 VARCHAR(50),f4 VARCHAR(50),f5 VARCHAR(50),f6 VARCHAR(50),f7 VARCHAR(50),f8 VARCHAR(50),f9 VARCHAR(50),f10 VARCHAR(50));`); err != nil {
			panic(err)
		}

		insert := "INSERT INTO dummy_table (f1,f2,f3,f4,f5,f6,f7,f8,f9,f10) VALUES "
		for i := 0; i < 9999; i += 1 {
			insert += fmt.Sprintf(`("%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")`, "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111")
			insert += ","
		}
		insert = insert[:len(insert)-1]

		if _, err := db.Exec(insert); err != nil {
			panic(err)
		}
	}

	for {
		f()
	}
}

[mattn]

for 10 minutes.

[wxsms]

Thanks for response. Forgot to mention, I've tested on a web server, for requests with same table data, memory do not increase after a while, but it's not going down anyway. for requests with different table datas, memory keep growing and go OOM killed finally.

[wxsms]

Looks like for this simple program, it's mem usage stops at some point after a while. But I think the key problem in this case is that, the big part of memory cost is never getting gc. It's more obvious in a webserver. In a real production server of mime which accepts various table structure and data, it's getting OOM killed very fast.

If I remove the db insert usage, just create the big insert string and let go, the mem usage drops soon and keep at a low level about 10mb.

[hallgrenx]

@wxsms could it be that defer db.Close() never runs? Try move it before the r.GET() as the main function is never returning.

func main() {
   r := gin.Default()
   kdb, err := sql.Open("sqlite3", "file::memory:")
   if err != nil {
	panic(err)
   }
   defer db.Close()

   r.GET("/run", func(c *gin.Context) {
                 if _, err := db.Exec(`CREATE TABLE dummy_table (id INT PRIMARY KEY,f1 VARCHAR(50),f2 VARCHAR(50),f3 VARCHAR(50),f4 VARCHAR(50),f5 VARCHAR(50),f6 VARCHAR(50),f7 VARCHAR(50),f8 VARCHAR(50),f9 VARCHAR(50),f10 VARCHAR(50));`); err != nil {
			panic(err)
		}

		insert := "INSERT INTO dummy_table (f1,f2,f3,f4,f5,f6,f7,f8,f9,f10) VALUES "
		for i := 0; i < 9999; i += 1 {
			insert += fmt.Sprintf(`("%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")`, "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111")
			insert += ","
		}
		insert = insert[:len(insert)-1]

		if _, err := db.Exec(insert); err != nil {
			panic(err)
		}
		c.String(200, "OK")
   })
   r.Run(":8080")
}

[mattn]

I figured out this behavior. This is not related on go-sqlite3.

package main

import (
	"database/sql"
	"fmt"

	_ "github.com/mattn/go-sqlite3"
)

var _ = fmt.Sprint

var insert string

func init() {
	insert = "INSERT INTO dummy_table (f1,f2,f3,f4,f5,f6,f7,f8,f9,f10) VALUES "
	for i := 0; i < 9999; i += 1 {
		insert += fmt.Sprintf(`("%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")`, "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111")
		insert += ","
	}
	insert = insert[:len(insert)-1]
}

func f() error {
	db, err := sql.Open("sqlite3", ":memory:")
	if err != nil {
		return err
	}
	defer db.Close()
	db.SetMaxOpenConns(1)
	db.SetMaxIdleConns(1)
	db.SetConnMaxIdleTime(0)
	db.SetConnMaxLifetime(0)

	if _, err := db.Exec(`CREATE TABLE dummy_table (id INT PRIMARY KEY,f1 VARCHAR(50),f2 VARCHAR(50),f3 VARCHAR(50),f4 VARCHAR(50),f5 VARCHAR(50),f6 VARCHAR(50),f7 VARCHAR(50),f8 VARCHAR(50),f9 VARCHAR(50),f10 VARCHAR(50));`); err != nil {
		return err
	}

	if _, err := db.Exec(insert); err != nil {
		return err
	}
	insert = ""
	return nil
}

func main() {
	for {
		fmt.Println(f())
	}
}

This code does not increse memory.

The reason of incresing memory in your code (and first my PoC) is simply an appending many values into variable insert.

[wxsms]

the insert = "" in your code makes it run nothing on 2+ rounds. maybe this is why memory doesnt increase in such case.

[wxsms]

I just updated my reproduce: https://github.com/wxsms/go-sqlite3-memleak , which create a insert string at the init function and reuse on every run. The memleak is still there. @mattn

[mattn]

Please try test without gin gonic.

package main

import (
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"runtime"
)

var insert string

func init() {
	insert = "INSERT INTO dummy_table (f1,f2,f3,f4,f5,f6,f7,f8,f9,f10) VALUES "
	for i := 0; i < 9999; i += 1 {
		insert += fmt.Sprintf(`("%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")`, "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111")
		insert += ","
	}
	insert = insert[:len(insert)-1]
}

func f() {
	db, err := sql.Open("sqlite3", "file::memory:")
	if err != nil {
		panic(err)
	}
	defer db.Close()

	if _, err := db.Exec(`CREATE TABLE dummy_table (id INT PRIMARY KEY,f1 VARCHAR(50),f2 VARCHAR(50),f3 VARCHAR(50),f4 VARCHAR(50),f5 VARCHAR(50),f6 VARCHAR(50),f7 VARCHAR(50),f8 VARCHAR(50),f9 VARCHAR(50),f10 VARCHAR(50));`); err != nil {
		panic(err)
	}
	if _, err := db.Exec(insert); err != nil {
		panic(err)
	}
}
func main() {
	for {
		f()
		runtime.GC()
	}
}

[mattn]

This is result of 10 minutes.

[mattn]

It is possible to be a memory leak of github.com/gin-gonic/gin.

[mattn]

20 minutes

[wxsms]

ok. I remove the gin framework. but as I mention before, the key problem is:

for requests with same table data, memory do not increase after a while, but it's not going down anyway

please see this code

package main

import (
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"runtime"
	"time"
)

var insert string

func init() {
	insert = "INSERT INTO dummy_table (f1,f2,f3,f4,f5,f6,f7,f8,f9,f10) VALUES "
	for i := 0; i < 9999; i += 1 {
		insert += fmt.Sprintf(`("%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")`, "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111")
		insert += ","
	}
	insert = insert[:len(insert)-1]
}

func f() {
	db, err := sql.Open("sqlite3", "file::memory:")
	if err != nil {
		panic(err)
	}
	defer db.Close()

	if _, err := db.Exec(`CREATE TABLE dummy_table (id INT PRIMARY KEY,f1 VARCHAR(50),f2 VARCHAR(50),f3 VARCHAR(50),f4 VARCHAR(50),f5 VARCHAR(50),f6 VARCHAR(50),f7 VARCHAR(50),f8 VARCHAR(50),f9 VARCHAR(50),f10 VARCHAR(50));`); err != nil {
		panic(err)
	}
	if _, err := db.Exec(insert); err != nil {
		panic(err)
	}
}
func main() {
	idx := 0
	for {
		fmt.Printf("round %d\n", idx)
		f()
		runtime.GC()
		idx++
		if idx == 5 {
                        // we sleep here to wait for gc
			time.Sleep(time.Hour * 111)
		}
	}
}

After 5 rounds of running, the memory is about 110mb, and never drops. Which behave the same as using gin webserver.

If we add sleep for each round after runtime.GC(), we can also see that the memory on startup is about 15mb, and every round running it increase ~20mb.

[mattn]

Calling runtime.GC does not necessarily trigger GC every time. GC will not be triggered until a certain threshold is reached.

[mattn]

This is a result of 5 minutes.

[mattn]

Please try this.

package main

import (
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"runtime"
)

var insert string

func init() {
	insert = "INSERT INTO dummy_table (f1,f2,f3,f4,f5,f6,f7,f8,f9,f10) VALUES "
	for i := 0; i < 9999; i += 1 {
		insert += fmt.Sprintf(`("%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")`, "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111", "1111111111")
		insert += ","
	}
	insert = insert[:len(insert)-1]
}

func f() {
	db, err := sql.Open("sqlite3", "file::memory:")
	if err != nil {
		panic(err)
	}
	defer db.Close()

	if _, err := db.Exec(`CREATE TABLE dummy_table (id INT PRIMARY KEY,f1 VARCHAR(50),f2 VARCHAR(50),f3 VARCHAR(50),f4 VARCHAR(50),f5 VARCHAR(50),f6 VARCHAR(50),f7 VARCHAR(50),f8 VARCHAR(50),f9 VARCHAR(50),f10 VARCHAR(50));`); err != nil {
		panic(err)
	}
	if _, err := db.Exec(insert); err != nil {
		panic(err)
	}
}
func main() {
	var m runtime.MemStats
	for {
		f()
		runtime.GC()

		runtime.ReadMemStats(&m)
		fmt.Printf("Alloc = %v\n", m.Alloc)
	}
}

You should understand that Go using memory for runtime too.

[wxsms]

Thank you for your patience in replying. My bad that forgot to mention: memory monitors in golang including pprof can not detect this leak. The heap alloc/inuse can not represent the real memory usage by the program. that's why I am using a docker container with docker stats to monitor the mem usage. could this be a leak on CGO side?