Segmentation fault (core dumped)

[microsoft]

Segmentation fault (core dumped)

[MaggieQi]

Currently, we only support np.float32 type. np.random.uniform will generate np.float64 data. Try to use np.random.uniform(low=0.5, high=13.3, size=(n,)).astype(np.float32)

[SueSu-Wish]

@MaggieQi I also got segmentation fault with np.float32 type.
python 2.7, ubuntu 14.04

import SPTAG
import numpy as np

n = 100
k = 3
r = 3

def testBuild(algo, distmethod, x, out):
   i = SPTAG.AnnIndex(algo, 'Float', x.shape[1])
   i.SetBuildParam("NumberOfThreads", '4')
   i.SetBuildParam("DistCalcMethod", distmethod)
   ret = i.Build(x.tobytes(), x.shape[0])
   i.Save(out)

def testBuildWithMetaData(algo, distmethod, x, s, out):
   i = SPTAG.AnnIndex(algo, 'Float', x.shape[1])
   i.SetBuildParam("NumberOfThreads", '4')
   i.SetBuildParam("DistCalcMethod", distmethod)
   if i.BuildWithMetaData(x.tobytes(), s, x.shape[0]):
       i.Save(out)

def testSearch(index, q, k):
   j = SPTAG.AnnIndex.Load(index)
   for t in range(q.shape[0]):
       result = j.Search(q[t].tobytes(), k)
       print (result[0]) # ids
       print (result[1]) # distances

def testSearchWithMetaData(index, q, k):
   j = SPTAG.AnnIndex.Load(index)
   j.SetSearchParam("MaxCheck", '1024')
   for t in range(q.shape[0]):
       result = j.SearchWithMetaData(q[t].tobytes(), k)
       print (result[0]) # ids
       print (result[1]) # distances
       print (result[2]) # metadata

def testAdd(index, x, out, algo, distmethod):
   if index != None:
       i = SPTAG.AnnIndex.Load(index)
   else:
       i = SPTAG.AnnIndex(algo, 'Float', x.shape[1])
   i.SetBuildParam("NumberOfThreads", '4')
   i.SetBuildParam("DistCalcMethod", distmethod)
   if i.Add(x.tobytes(), x.shape[0]):
       i.Save(out)

def testAddWithMetaData(index, x, s, out, algo, distmethod):
   if index != None:
       i = SPTAG.AnnIndex.Load(index)
   else:
       i = SPTAG.AnnIndex(algo, 'Float', x.shape[1])
   i = SPTAG.AnnIndex(algo, 'Float', x.shape[1])
   i.SetBuildParam("NumberOfThreads", '4')
   i.SetBuildParam("DistCalcMethod", distmethod)
   if i.AddWithMetaData(x.tobytes(), s, x.shape[0]):
       i.Save(out)

def testDelete(index, x, out):
   i = SPTAG.AnnIndex.Load(index)
   ret = i.Delete(x.tobytes(), x.shape[0])
   print (ret)
   i.Save(out)

def Test(algo, distmethod):
   x = np.ones((n, 10), dtype=np.float32) * np.reshape(np.arange(n, dtype=np.float32), (n, 1))
   q = np.ones((r, 10), dtype=np.float32) * np.reshape(np.arange(r, dtype=np.float32), (r, 1)) * 2
   print x
   print q
   m = ''
   for i in range(n):
       m += str(i) + '\n'

   print ("Build.............................")
   testBuild(algo, distmethod, x, 'testindices')
   testSearch('testindices', q, k)
   print ("Add.............................")
   testAdd('testindices', x, 'testindices', algo, distmethod)
   testSearch('testindices', q, k)
   print ("Delete.............................")
   testDelete('testindices', q, 'testindices')
   testSearch('testindices', q, k)

   print ("AddWithMetaData.............................")
   testAddWithMetaData(None, x, m, 'testindices', algo, distmethod)
   print ("Delete.............................")
   testSearchWithMetaData('testindices', q, k)
   testDelete('testindices', q, 'testindices')
   testSearchWithMetaData('testindices', q, k)

if __name__ == '__main__':
   Test('BKT', 'L2')
   Test('KDT', 'L2')

Build.............................
Setting NumberOfThreads with value 4
Setting DistCalcMethod with value L2
Start to build BKTree 1
1 BKTree built, 101 100
build RNG graph!
Refine 1 0%Refine RNG, graph acc:1
Refine 2 0%Refine RNG, graph acc:1
Build RNG Graph end!
[-1, -1, -1]
[3.4028234663852886e+38, 3.4028234663852886e+38, 3.4028234663852886e+38]
[-1, -1, -1]
[3.4028234663852886e+38, 3.4028234663852886e+38, 3.4028234663852886e+38]
[-1, -1, -1]
[3.4028234663852886e+38, 3.4028234663852886e+38, 3.4028234663852886e+38]
Add.............................
Setting NumberOfThreads with value 4
Setting DistCalcMethod with value L2
[-1, -1, -1]
[3.4028234663852886e+38, 3.4028234663852886e+38, 3.4028234663852886e+38]
[-1, -1, -1]
[3.4028234663852886e+38, 3.4028234663852886e+38, 3.4028234663852886e+38]
[-1, -1, -1]
[3.4028234663852886e+38, 3.4028234663852886e+38, 3.4028234663852886e+38]
Delete.............................
False
Segmentation fault (core dumped)

[0xhanh]

Currently, we only support np.float32 type. np.random.uniform will generate np.float64 data. Try to use np.random.uniform(low=0.5, high=13.3, size=(n,)).astype(np.float32)

Passed, thanks

[MaggieQi]

@SueSu-Wish from the log, it seems the index is not saved successfully.

[sruti25]

Trying to run IndexBuilder from command line like this:
./indexbuilder -d 1 -v Float -i ../all_question_embeddings.txt -o index/ -a BKT -t 4
but end up getting segmentation fault. I am using np.float32 as the data type.

Setting NumberOfThreads with value 4
Begin Subtask: 0, start offset position:0
Begin Subtask: 1, start offset position:2420000
Begin Subtask: 2, start offset position:4840000
Begin Subtask: 3, start offset position:7260000
Start to build BKTree 1
1 BKTree built, 4001 4000
build RNG graph!
Parallel TpTree Partition begin
Segmentation fault