Got an OOM when uploading 6GB file

[transloadit]

Got an OOM when uploading 6GB file

[Acconut]

FYI, I finally found the reason for this OOM exception: https://github.com/transloadit/go-sdk/blob/535a38a0e8e534cbd43a041541ab3d80a8667529/assembly.go#L202 This caused the multipart request body to be buffered in memory before sending it out. Luckily, I already have a solution for this.

[Acconut]

This is wired since the error occurs before the actual request has been sent off. Can be please try again with the latest version from the master branch?

[kvz]

I've tried again with

mkdir input output

cd  input
wget http://fixture.transloadit.com.s3.amazonaws.com/file/dennis-large.mp4
cd ..

curl http://releases.transloadit.com/transloadify/transloadify-linux-amd64-latest \
  -o ./transloadify && chmod 755 $_

echo '{
  "read": {
    "use": ":original",
    "robot": "/meta/read"
  }
}' > template.json

TRANSLOADIT_KEY=x \
TRANSLOADIT_SECRET=y \
./transloadify \
  -input="./input" \
  -output="./output" \
  -template-file="./template.json"

Same thing:

2015/10/13 16:46:21 Converting all files in '/root/input' and putting the result into '/root/output'.
2015/10/13 16:46:21 Using template file '/root/template.json' (read 1 steps).
fatal error: runtime: out of memory

goroutine 20 [running]:
runtime.throw(0x9306f7)
    /usr/local/go/src/pkg/runtime/panic.c:520 +0x69 fp=0x7f755395e538 sp=0x7f755395e520
runtime.SysMap(0xc3097c0000, 0x101680000, 0x427f00, 0x93bf58)
    /usr/local/go/src/pkg/runtime/mem_linux.c:147 +0x93 fp=0x7f755395e568 sp=0x7f755395e538
runtime.MHeap_SysAlloc(0x947f40, 0x101680000)
    /usr/local/go/src/pkg/runtime/malloc.goc:616 +0x15b fp=0x7f755395e5c0 sp=0x7f755395e568
MHeap_Grow(0x947f40, 0x80b40)
    /usr/local/go/src/pkg/runtime/mheap.c:319 +0x5d fp=0x7f755395e600 sp=0x7f755395e5c0
MHeap_AllocLocked(0x947f40, 0x80b3c, 0x0)
    /usr/local/go/src/pkg/runtime/mheap.c:222 +0x379 fp=0x7f755395e640 sp=0x7f755395e600
runtime.MHeap_Alloc(0x947f40, 0x80b3c, 0x10100000000)
    /usr/local/go/src/pkg/runtime/mheap.c:178 +0x7b fp=0x7f755395e668 sp=0x7f755395e640
largealloc(0xc200000001, 0x7f755395e718)
    /usr/local/go/src/pkg/runtime/malloc.goc:224 +0xa2 fp=0x7f755395e6b0 sp=0x7f755395e668
runtime.mallocgc(0x101678000, 0x67fd81, 0x1)
    /usr/local/go/src/pkg/runtime/malloc.goc:169 +0xb6 fp=0x7f755395e718 sp=0x7f755395e6b0
cnew(0x67fd80, 0x101678000, 0x7f7500000001)
    /usr/local/go/src/pkg/runtime/malloc.goc:836 +0xc1 fp=0x7f755395e738 sp=0x7f755395e718
runtime.cnewarray(0x67fd80, 0x101678000)
    /usr/local/go/src/pkg/runtime/malloc.goc:849 +0x3a fp=0x7f755395e758 sp=0x7f755395e738
makeslice1(0x671940, 0x101678000, 0x101678000, 0x7f755395e7b8)
    /usr/local/go/src/pkg/runtime/slice.goc:55 +0x4d fp=0x7f755395e770 sp=0x7f755395e758
runtime.makeslice(0x671940, 0x101678000, 0x101678000, 0x0, 0x101678000, 0x101678000)
    /usr/local/go/src/pkg/runtime/slice.goc:36 +0xb3 fp=0x7f755395e7a0 sp=0x7f755395e770
bytes.makeSlice(0x101678000, 0x0, 0x0, 0x0)
    /usr/local/go/src/pkg/bytes/buffer.go:191 +0x64 fp=0x7f755395e7d8 sp=0x7f755395e7a0
bytes.(*Buffer).grow(0xc208026620, 0x8000, 0xc208098000)
    /usr/local/go/src/pkg/bytes/buffer.go:99 +0x204 fp=0x7f755395e878 sp=0x7f755395e7d8
bytes.(*Buffer).Write(0xc208026620, 0xc208098000, 0x8000, 0x8000, 0xc20803a058, 0x0, 0x0)
    /usr/local/go/src/pkg/bytes/buffer.go:127 +0x56 fp=0x7f755395e8e0 sp=0x7f755395e878
mime/multipart.(*part).Write(0xc208042960, 0xc208098000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
    /usr/local/go/src/pkg/mime/multipart/writer.go:179 +0x148 fp=0x7f755395e958 sp=0x7f755395e8e0
io.Copy(0x7f7553ae6da0, 0xc208042960, 0x7f7553ae53f0, 0xc20803a058, 0x80b30000, 0x0, 0x0)
    /usr/local/go/src/pkg/io/io.go:355 +0x27b fp=0x7f755395ea20 sp=0x7f755395e958
github.com/transloadit/go-sdk.(*Assembly).makeRequest(0xc20801a780, 0x42664f, 0x0, 0x0)
    /gopath/src/github.com/transloadit/go-sdk/assembly.go:202 +0x440 fp=0x7f755395ebe0 sp=0x7f755395ea20
github.com/transloadit/go-sdk.(*Assembly).Upload(0xc20801a780, 0xc20803a068, 0x0, 0x0)
    /gopath/src/github.com/transloadit/go-sdk/assembly.go:156 +0x4a fp=0x7f755395eca8 sp=0x7f755395ebe0
github.com/transloadit/go-sdk.(*Watcher).processFile(0xc20801a6e0, 0xc208042640, 0x1c)
    /gopath/src/github.com/transloadit/go-sdk/watch.go:172 +0x5c1 fp=0x7f755395ef90 sp=0x7f755395eca8
runtime.goexit()
    /usr/local/go/src/pkg/runtime/proc.c:1445 fp=0x7f755395ef98 sp=0x7f755395ef90
created by github.com/transloadit/go-sdk.(*Watcher).processDir
    /gopath/src/github.com/transloadit/go-sdk/watch.go:139 +0x267

goroutine 16 [select]:
main.main()
    /gopath/src/github.com/transloadit/transloadify/transloadify.go:104 +0xc57

goroutine 19 [finalizer wait]:
runtime.park(0x417e50, 0x935df0, 0x934089)
    /usr/local/go/src/pkg/runtime/proc.c:1369 +0x89
runtime.parkunlock(0x935df0, 0x934089)
    /usr/local/go/src/pkg/runtime/proc.c:1385 +0x3b
runfinq()
    /usr/local/go/src/pkg/runtime/mgc0.c:2644 +0xcf
runtime.goexit()
    /usr/local/go/src/pkg/runtime/proc.c:1445

goroutine 17 [syscall]:
runtime.goexit()
    /usr/local/go/src/pkg/runtime/proc.c:1445

[kvz]

No longer relevant since we switched to Node

[kvz]

That's good to know!