Enhance testing framework based on kubernetes (#935)

* Update framework

* remove files

* Remove files

* Remove ann-acc cases && Update java-sdk cases

tests/milvus-java-test/pom.xml

@@ -99,7 +99,7 @@
         <dependency>
             <groupId>io.milvus</groupId>
             <artifactId>milvus-sdk-java</artifactId>
-            <version>0.3.0</version>
+            <version>0.4.1-SNAPSHOT</version>
         </dependency>
 
 <!--        <dependency>-->
@@ -134,4 +134,4 @@
 
     </dependencies>
 
-</project>
+</project>

tests/milvus-java-test/src/main/java/com/MainClass.java

@@ -15,7 +15,7 @@ import java.util.List;
 
 public class MainClass {
     private static String host = "127.0.0.1";
-    private static String port = "19530";
+    private static String port = "19532";
     private int index_file_size = 50;
     public int dimension = 128;
 

tests/milvus-java-test/src/main/java/com/Partition.java

@@ -0,0 +1,205 @@
+package com;
+
+import io.milvus.client.MilvusClient;
+import io.milvus.client.Response;
+import io.milvus.client.ShowPartitionsResponse;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+
+public class Partition {
+    int dimension = 128;
+
+    public List<List<Float>> gen_vectors(Integer nb) {
+        List<List<Float>> xb = new LinkedList<>();
+        Random random = new Random();
+        for (int i = 0; i < nb; ++i) {
+            LinkedList<Float> vector = new LinkedList<>();
+            for (int j = 0; j < dimension; j++) {
+                vector.add(random.nextFloat());
+            }
+            xb.add(vector);
+        }
+        return xb;
+    }
+
+    // ----------------------------- create partition cases in ---------------------------------
+
+    // create partition
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_create_partition(MilvusClient client, String tableName) {
+        String partitionName = RandomStringUtils.randomAlphabetic(10);
+        String tag = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        Response createpResponse = client.createPartition(partition);
+        assert (createpResponse.ok());
+        // show partitions
+        List<io.milvus.client.Partition> partitions = client.showPartitions(tableName).getPartitionList();
+        System.out.println(partitions);
+        List<String> partitionNames = new ArrayList<>();
+        for (int i=0; i<partitions.size(); ++i) {
+            partitionNames.add(partitions.get(i).getPartitionName());
+        }
+        Assert.assertTrue(partitionNames.contains(partitionName));
+    }
+
+    // create partition, partition existed
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_create_partition_name_existed(MilvusClient client, String tableName) {
+        String partitionName = RandomStringUtils.randomAlphabetic(10);
+        String tag = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        Response createpResponse = client.createPartition(partition);
+        assert (createpResponse.ok());
+        String newTag = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition newPartition = new io.milvus.client.Partition.Builder(tableName, partitionName, newTag).build();
+        Response newCreatepResponse = client.createPartition(newPartition);
+        assert (!newCreatepResponse.ok());
+    }
+
+    // create partition, tag name existed
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_create_partition_tag_name_existed(MilvusClient client, String tableName) {
+        String partitionName = RandomStringUtils.randomAlphabetic(10);
+        String tag = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        Response createpResponse = client.createPartition(partition);
+        assert (createpResponse.ok());
+        io.milvus.client.Partition newPartition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        Response newCreatepResponse = client.createPartition(newPartition);
+        assert (!newCreatepResponse.ok());
+    }
+
+    // ----------------------------- drop partition cases in ---------------------------------
+
+    // drop a partition created before, drop by partition name
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_drop_partition(MilvusClient client, String tableName) {
+        String partitionName = RandomStringUtils.randomAlphabetic(10);
+        String tag = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        Response createpResponse = client.createPartition(partition);
+        assert (createpResponse.ok());
+        Response response = client.dropPartition(partitionName);
+        assert (response.ok());
+        // show partitions
+        int length = client.showPartitions(tableName).getPartitionList().size();
+        Assert.assertEquals(length, 0);
+    }
+
+    // drop a partition repeat created before, drop by partition name
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_drop_partition_repeat(MilvusClient client, String tableName) throws InterruptedException {
+        String partitionName = RandomStringUtils.randomAlphabetic(10);
+        String tag = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        Response createpResponse = client.createPartition(partition);
+        assert (createpResponse.ok());
+        Response response = client.dropPartition(partitionName);
+        assert (response.ok());
+        Thread.currentThread().sleep(2000);
+        Response newResponse = client.dropPartition(partitionName);
+        assert (!newResponse.ok());
+    }
+
+    // drop a partition created before, drop by tag
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_drop_partition_with_tag(MilvusClient client, String tableName) {
+        String partitionName = RandomStringUtils.randomAlphabetic(10);
+        String tag = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        Response createpResponse = client.createPartition(partition);
+        assert (createpResponse.ok());
+        Response response = client.dropPartition(tableName, tag);
+        assert (response.ok());
+        // show partitions
+        int length = client.showPartitions(tableName).getPartitionList().size();
+        Assert.assertEquals(length, 0);
+    }
+
+    // drop a partition not created before
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_drop_partition_not_existed(MilvusClient client, String tableName) {
+        String partitionName = RandomStringUtils.randomAlphabetic(10);
+        String tag = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        Response createpResponse = client.createPartition(partition);
+        assert(createpResponse.ok());
+        String newPartitionName = RandomStringUtils.randomAlphabetic(10);
+        Response response = client.dropPartition(newPartitionName);
+        assert(!response.ok());
+    }
+
+    // drop a partition not created before
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_drop_partition_tag_not_existed(MilvusClient client, String tableName) {
+        String partitionName = RandomStringUtils.randomAlphabetic(10);
+        String tag = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        Response createpResponse = client.createPartition(partition);
+        assert(createpResponse.ok());
+        String newTag = RandomStringUtils.randomAlphabetic(10);
+        Response response = client.dropPartition(tableName, newTag);
+        assert(!response.ok());
+    }
+
+    // ----------------------------- show partitions cases in ---------------------------------
+
+    // create partition, then show partitions
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_show_partitions(MilvusClient client, String tableName) {
+        String partitionName = RandomStringUtils.randomAlphabetic(10);
+        String tag = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        Response createpResponse = client.createPartition(partition);
+        assert (createpResponse.ok());
+        ShowPartitionsResponse response = client.showPartitions(tableName);
+        assert (response.getResponse().ok());
+        List<String> partitionNames = new ArrayList<>();
+        for (int i=0; i<response.getPartitionList().size(); ++i) {
+            partitionNames.add(response.getPartitionList().get(i).getPartitionName());
+            if (response.getPartitionList().get(i).getPartitionName() == partitionName) {
+                Assert.assertTrue(response.getPartitionList().get(i).getTableName() == tableName);
+                Assert.assertTrue(response.getPartitionList().get(i).getTag() == tag);
+            }
+        }
+        Assert.assertTrue(partitionNames.contains(partitionName));
+    }
+
+    // create multi partition, then show partitions
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_show_partitions_multi(MilvusClient client, String tableName) {
+        String partitionName = RandomStringUtils.randomAlphabetic(10);
+        String tag = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        Response createpResponse = client.createPartition(partition);
+        assert (createpResponse.ok());
+        String newPartitionName = RandomStringUtils.randomAlphabetic(10);
+        String tagNew = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition newPartition = new io.milvus.client.Partition.Builder(tableName, newPartitionName, tagNew).build();
+        Response newCreatepResponse = client.createPartition(newPartition);
+        assert (newCreatepResponse.ok());
+        ShowPartitionsResponse response = client.showPartitions(tableName);
+        assert (response.getResponse().ok());
+        List<String> partitionNames = response.getPartitionNameList();
+//        for (int i=0; i<response.getPartitionList().size(); ++i) {
+//            partitionNames.add(response.getPartitionList().get(i).getPartitionName());
+//            if (response.getPartitionList().get(i).getPartitionName() == newPartitionName) {
+//                Assert.assertTrue(response.getPartitionList().get(i).getTableName() == tableName);
+//                Assert.assertTrue(response.getPartitionList().get(i).getTag() == tagNew);
+//            }
+//        }
+        System.out.println(partitionNames);
+        Assert.assertTrue(partitionNames.contains(partitionName));
+        Assert.assertTrue(partitionNames.contains(newPartitionName));
+        List<String> tagNames = response.getPartitionTagList();
+        Assert.assertTrue(tagNames.contains(tag));
+        Assert.assertTrue(tagNames.contains(tagNew));
+    }
+
+}

tests/milvus-java-test/src/main/java/com/TestAddVectors.java

@@ -1,8 +1,7 @@
 package com;
 
-import io.milvus.client.InsertParam;
-import io.milvus.client.InsertResponse;
-import io.milvus.client.MilvusClient;
+import io.milvus.client.*;
+import org.apache.commons.lang3.RandomStringUtils;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
@@ -12,6 +11,7 @@ import java.util.stream.Stream;
 
 public class TestAddVectors {
     int dimension = 128;
+    String tag = "tag";
 
     public List<List<Float>> gen_vectors(Integer nb) {
         List<List<Float>> xb = new LinkedList<>();
@@ -28,7 +28,7 @@ public class TestAddVectors {
 
     @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
     public void test_add_vectors_table_not_existed(MilvusClient client, String tableName) throws InterruptedException {
-        int nb = 10000;
+        int nb = 1000;
         List<List<Float>> vectors = gen_vectors(nb);
         String tableNameNew = tableName + "_";
         InsertParam insertParam = new InsertParam.Builder(tableNameNew, vectors).build();
@@ -47,7 +47,7 @@ public class TestAddVectors {
 
     @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
     public void test_add_vectors(MilvusClient client, String tableName) throws InterruptedException {
-        int nb = 10000;
+        int nb = 1000;
         List<List<Float>> vectors = gen_vectors(nb);
         InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
         InsertResponse res = client.insert(insertParam);
@@ -79,7 +79,7 @@ public class TestAddVectors {
 
     @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
     public void test_add_vectors_with_ids(MilvusClient client, String tableName) throws InterruptedException {
-        int nb = 10000;
+        int nb = 1000;
         List<List<Float>> vectors = gen_vectors(nb);
         // Add vectors with ids
         List<Long> vectorIds;
@@ -111,7 +111,7 @@ public class TestAddVectors {
 
     @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
     public void test_add_vectors_with_invalid_dimension(MilvusClient client, String tableName) {
-        int nb = 10000;
+        int nb = 1000;
         List<List<Float>> vectors = gen_vectors(nb);
         vectors.get(0).add((float) 0);
         InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
@@ -121,7 +121,7 @@ public class TestAddVectors {
 
     @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
     public void test_add_vectors_with_invalid_vectors(MilvusClient client, String tableName) {
-        int nb = 10000;
+        int nb = 1000;
         List<List<Float>> vectors = gen_vectors(nb);
         vectors.set(0, new ArrayList<>());
         InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
@@ -147,4 +147,53 @@ public class TestAddVectors {
         Assert.assertEquals(client.getTableRowCount(tableName).getTableRowCount(), nb * loops);
     }
 
+    // ----------------------------- partition cases in Insert ---------------------------------
+    // Add vectors into table with given tag
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_add_vectors_partition(MilvusClient client, String tableName) throws InterruptedException {
+        int nb = 1000;
+        List<List<Float>> vectors = gen_vectors(nb);
+        String partitionName = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        Response createpResponse = client.createPartition(partition);
+        assert(createpResponse.ok());
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(tag).build();
+        InsertResponse res = client.insert(insertParam);
+        assert(res.getResponse().ok());
+        Thread.currentThread().sleep(1000);
+        // Assert table row count
+        Assert.assertEquals(client.getTableRowCount(tableName).getTableRowCount(), nb);
+    }
+
+    // Add vectors into table, which tag not existed
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_add_vectors_partition_tag_not_existed(MilvusClient client, String tableName) {
+        int nb = 1000;
+        String newTag = RandomStringUtils.randomAlphabetic(10);
+        List<List<Float>> vectors = gen_vectors(nb);
+        String partitionName = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        Response createpResponse = client.createPartition(partition);
+        assert(createpResponse.ok());
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(newTag).build();
+        InsertResponse res = client.insert(insertParam);
+        assert(!res.getResponse().ok());
+    }
+
+    // Create table, add vectors into table
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_add_vectors_partition_A(MilvusClient client, String tableName) throws InterruptedException {
+        int nb = 1000;
+        List<List<Float>> vectors = gen_vectors(nb);
+        String partitionName = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        Response createpResponse = client.createPartition(partition);
+        assert(createpResponse.ok());
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        InsertResponse res = client.insert(insertParam);
+        assert(res.getResponse().ok());
+        Thread.currentThread().sleep(1000);
+        // Assert table row count
+        Assert.assertEquals(client.getTableRowCount(tableName).getTableRowCount(), nb);
+    }
 }

tests/milvus-java-test/src/main/java/com/TestConnect.java

@@ -22,12 +22,13 @@ public class TestConnect {
     @Test(dataProvider = "DefaultConnectArgs", dataProviderClass = MainClass.class)
     public void test_connect_repeat(String host, String port) {
         MilvusGrpcClient client = new MilvusGrpcClient();
-        ConnectParam connectParam = new ConnectParam.Builder()
-                .withHost(host)
-                .withPort(port)
-                .build();
+
         Response res = null;
         try {
+            ConnectParam connectParam = new ConnectParam.Builder()
+                    .withHost(host)
+                    .withPort(port)
+                    .build();
             res = client.connect(connectParam);
             res = client.connect(connectParam);
         } catch (ConnectFailedException e) {
@@ -40,14 +41,14 @@ public class TestConnect {
     @Test(dataProvider="InvalidConnectArgs")
     public void test_connect_invalid_connect_args(String ip, String port) {
         MilvusClient client = new MilvusGrpcClient();
-        ConnectParam connectParam = new ConnectParam.Builder()
-                .withHost(ip)
-                .withPort(port)
-                .build();
         Response res = null;
         try {
+            ConnectParam connectParam = new ConnectParam.Builder()
+                    .withHost(ip)
+                    .withPort(port)
+                    .build();
             res = client.connect(connectParam);
-        } catch (ConnectFailedException e) {
+        } catch (Exception e) {
             e.printStackTrace();
         }
         Assert.assertEquals(res, null);

tests/milvus-java-test/src/main/java/com/TestIndex.java

@@ -134,7 +134,7 @@ public class TestIndex {
 
     @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
     public void test_create_index_IVFSQ8H(MilvusClient client, String tableName) throws InterruptedException {
-        IndexType indexType = IndexType.IVF_SQ8H;
+        IndexType indexType = IndexType.IVF_SQ8_H;
         List<List<Float>> vectors = gen_vectors(nb);
         InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
         client.insert(insertParam);

tests/milvus-java-test/src/main/java/com/TestMix.java

@@ -120,6 +120,31 @@ public class TestMix {
         Assert.assertEquals(getTableRowCountResponse.getTableRowCount(), thread_num * nb);
     }
 
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_add_vectors_partition_threads(MilvusClient client, String tableName) throws InterruptedException {
+        int thread_num = 10;
+        String tag = RandomStringUtils.randomAlphabetic(10);
+        String partitionName = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        client.createPartition(partition);
+        List<List<Float>> vectors = gen_vectors(nb,false);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(tag).build();
+        ForkJoinPool executor = new ForkJoinPool();
+        for (int i = 0; i < thread_num; i++) {
+            executor.execute(
+                    () -> {
+                        InsertResponse res_insert = client.insert(insertParam);
+                        assert (res_insert.getResponse().ok());
+                    });
+        }
+        executor.awaitQuiescence(100, TimeUnit.SECONDS);
+        executor.shutdown();
+
+        Thread.sleep(2000);
+        GetTableRowCountResponse getTableRowCountResponse = client.getTableRowCount(tableName);
+        Assert.assertEquals(getTableRowCountResponse.getTableRowCount(), thread_num * nb);
+    }
+
     @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
     public void test_add_index_vectors_threads(MilvusClient client, String tableName) throws InterruptedException {
         int thread_num = 50;

tests/milvus-java-test/src/main/java/com/TestPS.java

@@ -0,0 +1,115 @@
+package com;
+
+import io.milvus.client.*;
+import org.apache.commons.cli.*;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class TestPS {
+    private static int dimension = 128;
+    private static String host = "192.168.1.101";
+    private static String port = "19530";
+
+    public static void setHost(String host) {
+        TestPS.host = host;
+    }
+
+    public static void setPort(String port) {
+        TestPS.port = port;
+    }
+
+    public static List<Float> normalize(List<Float> w2v){
+        float squareSum = w2v.stream().map(x -> x * x).reduce((float) 0, Float::sum);
+        final float norm = (float) Math.sqrt(squareSum);
+        w2v = w2v.stream().map(x -> x / norm).collect(Collectors.toList());
+        return w2v;
+    }
+
+    public static List<List<Float>> gen_vectors(int nb, boolean norm) {
+        List<List<Float>> xb = new ArrayList<>();
+        Random random = new Random();
+        for (int i = 0; i < nb; ++i) {
+            List<Float> vector = new ArrayList<>();
+            for (int j = 0; j < dimension; j++) {
+                vector.add(random.nextFloat());
+            }
+            if (norm == true) {
+                vector = normalize(vector);
+            }
+            xb.add(vector);
+        }
+        return xb;
+    }
+
+    public static void main(String[] args) throws ConnectFailedException {
+        int nq = 5;
+        int nb = 1;
+        int nprobe = 32;
+        int top_k = 10;
+        int loops = 100000;
+//        int index_file_size = 1024;
+        String tableName = "sift_1b_2048_128_l2";
+
+        CommandLineParser parser = new DefaultParser();
+        Options options = new Options();
+        options.addOption("h", "host", true, "milvus-server hostname/ip");
+        options.addOption("p", "port", true, "milvus-server port");
+        try {
+            CommandLine cmd = parser.parse(options, args);
+            String host = cmd.getOptionValue("host");
+            if (host != null) {
+                setHost(host);
+            }
+            String port = cmd.getOptionValue("port");
+            if (port != null) {
+                setPort(port);
+            }
+            System.out.println("Host: "+host+", Port: "+port);
+        }
+        catch(ParseException exp) {
+            System.err.println("Parsing failed.  Reason: " + exp.getMessage() );
+        }
+
+        List<List<Float>> vectors = gen_vectors(nb, true);
+        List<List<Float>> queryVectors = gen_vectors(nq, true);
+        MilvusClient client = new MilvusGrpcClient();
+        ConnectParam connectParam = new ConnectParam.Builder()
+                .withHost(host)
+                .withPort(port)
+                .build();
+        client.connect(connectParam);
+//        String tableName = RandomStringUtils.randomAlphabetic(10);
+//        TableSchema tableSchema = new TableSchema.Builder(tableName, dimension)
+//                .withIndexFileSize(index_file_size)
+//                .withMetricType(MetricType.IP)
+//                .build();
+//        Response res = client.createTable(tableSchema);
+        List<Long> vectorIds;
+        vectorIds = Stream.iterate(0L, n -> n)
+                .limit(nb)
+                .collect(Collectors.toList());
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withVectorIds(vectorIds).build();
+        ForkJoinPool executor_search = new ForkJoinPool();
+        for (int i = 0; i < loops; i++) {
+            executor_search.execute(
+                    () -> {
+                        InsertResponse res_insert = client.insert(insertParam);
+                        assert (res_insert.getResponse().ok());
+                        System.out.println("In insert");
+                        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(nprobe).withTopK(top_k).build();
+                        SearchResponse res_search = client.search(searchParam);
+                        assert (res_search.getResponse().ok());
+                    });
+        }
+        executor_search.awaitQuiescence(300, TimeUnit.SECONDS);
+        executor_search.shutdown();
+        GetTableRowCountResponse getTableRowCountResponse = client.getTableRowCount(tableName);
+        System.out.println(getTableRowCountResponse.getTableRowCount());
+    }
+}

tests/milvus-java-test/src/main/java/com/TestSearchVectors.java

@@ -1,10 +1,12 @@
 package com;
 
 import io.milvus.client.*;
+import org.apache.commons.lang3.RandomStringUtils;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
 import java.util.*;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -53,7 +55,7 @@ public class TestSearchVectors {
     }
 
     @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
-    public void test_search_table_not_existed(MilvusClient client, String tableName) throws InterruptedException {
+    public void test_search_table_not_existed(MilvusClient client, String tableName)  {
         String tableNameNew = tableName + "_";
         int nq = 5;
         int nb = 100;
@@ -65,7 +67,7 @@ public class TestSearchVectors {
     }
 
     @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
-    public void test_search_index_IVFLAT(MilvusClient client, String tableName) throws InterruptedException {
+    public void test_search_index_IVFLAT(MilvusClient client, String tableName)  {
         IndexType indexType = IndexType.IVFLAT;
         int nq = 5;
         List<List<Float>> vectors = gen_vectors(nb, false);
@@ -84,7 +86,7 @@ public class TestSearchVectors {
     }
 
     @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
-    public void test_search_ids_IVFLAT(MilvusClient client, String tableName) throws InterruptedException {
+    public void test_search_ids_IVFLAT(MilvusClient client, String tableName)  {
         IndexType indexType = IndexType.IVFLAT;
         int nq = 5;
         List<List<Float>> vectors = gen_vectors(nb, true);
@@ -121,7 +123,7 @@ public class TestSearchVectors {
     }
 
     @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
-    public void test_search_distance_IVFLAT(MilvusClient client, String tableName) throws InterruptedException {
+    public void test_search_distance_IVFLAT(MilvusClient client, String tableName)  {
         IndexType indexType = IndexType.IVFLAT;
         int nq = 5;
         List<List<Float>> vectors = gen_vectors(nb, true);
@@ -144,7 +146,120 @@ public class TestSearchVectors {
     }
 
     @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
-    public void test_search_index_IVFSQ8(MilvusClient client, String tableName) throws InterruptedException {
+    public void test_search_distance_partition(MilvusClient client, String tableName) {
+        IndexType indexType = IndexType.IVFLAT;
+        int nq = 5;
+        String tag = RandomStringUtils.randomAlphabetic(10);
+        List<List<Float>> vectors = gen_vectors(nb, true);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        String partitionName = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        client.createPartition(partition);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(tag).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        client.createIndex(createIndexParam);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build();
+        List<List<SearchResponse.QueryResult>> res_search = client.search(searchParam).getQueryResultsList();
+        double distance = res_search.get(0).get(0).getDistance();
+        if (tableName.startsWith("L2")) {
+            Assert.assertEquals(distance, 0.0, epsilon);
+        }else if (tableName.startsWith("IP")) {
+            Assert.assertEquals(distance, 1.0, epsilon);
+        }
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_distance_partition_not_exited(MilvusClient client, String tableName) {
+        IndexType indexType = IndexType.IVFLAT;
+        int nq = 5;
+        String tag = RandomStringUtils.randomAlphabetic(10);
+        List<List<Float>> vectors = gen_vectors(nb, true);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        String partitionName = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        client.createPartition(partition);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(tag).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        client.createIndex(createIndexParam);
+        String tagNew = RandomStringUtils.randomAlphabetic(10);
+        List<String> queryTags = new ArrayList<>();
+        queryTags.add(tagNew);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withPartitionTags(queryTags).build();
+        SearchResponse res_search = client.search(searchParam);
+        assert (res_search.getResponse().ok());
+        Assert.assertEquals(res_search.getQueryResultsList().size(), 0);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_distance_partition_empty(MilvusClient client, String tableName) {
+        IndexType indexType = IndexType.IVFLAT;
+        int nq = 5;
+        String tag = RandomStringUtils.randomAlphabetic(10);
+        List<List<Float>> vectors = gen_vectors(nb, true);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        String partitionName = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        client.createPartition(partition);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(tag).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        client.createIndex(createIndexParam);
+        String tagNew = "";
+        List<String> queryTags = new ArrayList<>();
+        queryTags.add(tagNew);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withPartitionTags(queryTags).build();
+        SearchResponse res_search = client.search(searchParam);
+        assert (!res_search.getResponse().ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_distance_partition_A(MilvusClient client, String tableName) throws InterruptedException {
+//        IndexType indexType = IndexType.IVFLAT;
+        String tag = RandomStringUtils.randomAlphabetic(10);
+        String tagNew = RandomStringUtils.randomAlphabetic(10);
+        List<List<Float>> vectors = gen_vectors(nb, true);
+        List<List<Float>> vectorsNew = gen_vectors(nb, true);
+        String partitionName = RandomStringUtils.randomAlphabetic(10);
+        String partitionNameNew = RandomStringUtils.randomAlphabetic(10);
+        io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build();
+        io.milvus.client.Partition partitionNew = new io.milvus.client.Partition.Builder(tableName, partitionNameNew, tagNew).build();
+        client.createPartition(partition);
+        client.createPartition(partitionNew);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(tag).build();
+        InsertResponse res = client.insert(insertParam);
+        System.out.println(res.getVectorIds().size());
+        InsertParam insertParamNew = new InsertParam.Builder(tableName, vectorsNew).withPartitionTag(tagNew).build();
+        InsertResponse resNew = client.insert(insertParamNew);
+        TimeUnit.SECONDS.sleep(2);
+        System.out.println(resNew.getVectorIds().size());
+        List<String> queryTags = new ArrayList<>();
+        queryTags.add(tag);
+        List<List<Float>> queryVectors;
+        queryVectors = vectors.subList(0,2);
+        queryVectors.add(vectorsNew.get(0));
+        System.out.println(queryVectors.size());
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withPartitionTags(queryTags).build();
+        List<List<Long>> res_search = client.search(searchParam).getResultIdsList();
+        System.out.println(res_search.get(0));
+        System.out.println(res.getVectorIds());
+//        System.out.println(res_search.get(2));
+        Assert.assertTrue(res.getVectorIds().containsAll(res_search.get(0)));
+        Assert.assertTrue(resNew.getVectorIds().contains(res_search.get(2)));
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_index_IVFSQ8(MilvusClient client, String tableName)  {
         IndexType indexType = IndexType.IVF_SQ8;
         int nq = 5;
         List<List<Float>> vectors = gen_vectors(nb, false);
@@ -178,7 +293,7 @@ public class TestSearchVectors {
     }
 
     @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
-    public void test_search_distance_IVFSQ8(MilvusClient client, String tableName) throws InterruptedException {
+    public void test_search_distance_IVFSQ8(MilvusClient client, String tableName)  {
         IndexType indexType = IndexType.IVF_SQ8;
         int nq = 5;
         int nb = 1000;
@@ -192,7 +307,7 @@ public class TestSearchVectors {
         CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
         client.createIndex(createIndexParam);
         SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build();
-        List<List<Double>> res_search = client.search(searchParam).getResultDistancesList();
+        List<List<Float>> res_search = client.search(searchParam).getResultDistancesList();
         for (int i = 0; i < nq; i++) {
             double distance = res_search.get(i).get(0);
             System.out.println(distance);
@@ -205,7 +320,7 @@ public class TestSearchVectors {
     }
 
     @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
-    public void test_search_index_FLAT(MilvusClient client, String tableName) throws InterruptedException {
+    public void test_search_index_FLAT(MilvusClient client, String tableName)  {
         IndexType indexType = IndexType.FLAT;
         int nq = 5;
         List<List<Float>> vectors = gen_vectors(nb, false);
@@ -274,7 +389,7 @@ public class TestSearchVectors {
     }
 
     @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
-    public void test_search_distance_FLAT(MilvusClient client, String tableName) throws InterruptedException {
+    public void test_search_distance_FLAT(MilvusClient client, String tableName)  {
         IndexType indexType = IndexType.FLAT;
         int nq = 5;
         List<List<Float>> vectors = gen_vectors(nb, true);
@@ -297,7 +412,7 @@ public class TestSearchVectors {
     }
 
     @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
-    public void test_search_invalid_n_probe(MilvusClient client, String tableName) throws InterruptedException {
+    public void test_search_invalid_n_probe(MilvusClient client, String tableName)  {
         IndexType indexType = IndexType.IVF_SQ8;
         int nq = 5;
         int n_probe_new = 0;
@@ -316,7 +431,7 @@ public class TestSearchVectors {
     }
 
     @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
-    public void test_search_invalid_top_k(MilvusClient client, String tableName) throws InterruptedException {
+    public void test_search_invalid_top_k(MilvusClient client, String tableName)  {
         IndexType indexType = IndexType.IVF_SQ8;
         int nq = 5;
         int top_k_new = 0;
@@ -353,7 +468,7 @@ public class TestSearchVectors {
 //    }
 
     @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
-    public void test_search_index_range(MilvusClient client, String tableName) throws InterruptedException {
+    public void test_search_index_range(MilvusClient client, String tableName)  {
         IndexType indexType = IndexType.IVF_SQ8;
         int nq = 5;
         List<List<Float>> vectors = gen_vectors(nb, false);

tests/milvus_ann_acc/.gitignore

@@ -1,2 +0,0 @@
-__pycache__/
-logs/

tests/milvus_ann_acc/README.md

@@ -1,21 +0,0 @@
-# Requirements
-
-- python 3.6+
-- pip install -r requirements.txt
-
-# How to use this Test Project
-
-This project is used to test search accuracy based on the given datasets (https://github.com/erikbern/ann-benchmarks#data-sets)
-
-1. start your milvus server
-2. update your test configuration in test.py
-3. run command
-
-```shell
-python test.py
-```
-
-# Contribution getting started
-
-- Follow PEP-8 for naming and black for formatting.
-

tests/milvus_ann_acc/ci/jenkinsfile/acc_test.groovy

@@ -1,16 +0,0 @@
-timeout(time: 7200, unit: 'MINUTES') {
-    try {
-        dir ("milvu_ann_acc") {
-            print "Git clone url: ${TEST_URL}:${TEST_BRANCH}"
-            checkout([$class: 'GitSCM', branches: [[name: "${TEST_BRANCH}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "${TEST_URL}", name: 'origin', refspec: "+refs/heads/${TEST_BRANCH}:refs/remotes/origin/${TEST_BRANCH}"]]])
-            print "Install requirements"
-            sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
-            // sleep(120000)
-            sh "python3 main.py --suite=${params.SUITE} --host=acc-test-${env.JOB_NAME}-${env.BUILD_NUMBER}-engine.milvus.svc.cluster.local --port=19530"
-        }
-    } catch (exc) {
-        echo 'Milvus Ann Accuracy Test Failed !'
-        throw exc
-    }
-}
-

tests/milvus_ann_acc/ci/jenkinsfile/cleanup.groovy

@@ -1,13 +0,0 @@
-try {
-    def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true
-    if (!result) {
-        sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
-    }
-} catch (exc) {
-    def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true
-    if (!result) {
-        sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
-    }
-    throw exc
-}
-

tests/milvus_ann_acc/ci/jenkinsfile/deploy_default_server.groovy

@@ -1,22 +0,0 @@
-timeout(time: 30, unit: 'MINUTES') {
-    try {
-        dir ("milvus") {
-            sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
-            sh 'helm repo update'
-            checkout([$class: 'GitSCM', branches: [[name: "${HELM_BRANCH}"]], userRemoteConfigs: [[url: "${HELM_URL}", name: 'origin', refspec: "+refs/heads/${HELM_BRANCH}:refs/remotes/origin/${HELM_BRANCH}"]]])
-            dir ("milvus") {
-                sh "helm install --wait --timeout 300 --set engine.image.tag=${IMAGE_TAG} --set expose.type=clusterIP --name acc-test-${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/sqlite_${params.IMAGE_TYPE}_values.yaml -f ci/filebeat/values.yaml --namespace milvus --version ${HELM_BRANCH} ."
-            }
-        }
-        // dir ("milvus") {
-        //     checkout([$class: 'GitSCM', branches: [[name: "${env.SERVER_BRANCH}"]], userRemoteConfigs: [[url: "${env.SERVER_URL}", name: 'origin', refspec: "+refs/heads/${env.SERVER_BRANCH}:refs/remotes/origin/${env.SERVER_BRANCH}"]]])
-        //     dir ("milvus") {
-        //         load "ci/jenkins/step/deploySingle2Dev.groovy"
-        //     }
-        // }
-    } catch (exc) {
-        echo 'Deploy Milvus Server Failed !'
-        throw exc
-    }
-}
-

tests/milvus_ann_acc/client.py

@@ -1,146 +0,0 @@
-import pdb
-import random
-import logging
-import json
-import time, datetime
-from multiprocessing import Process
-import numpy
-import sklearn.preprocessing
-from milvus import Milvus, IndexType, MetricType
-
-logger = logging.getLogger("milvus_acc.client")
-
-SERVER_HOST_DEFAULT = "127.0.0.1"
-SERVER_PORT_DEFAULT = 19530
-
-
-def time_wrapper(func):
-    """
-    This decorator prints the execution time for the decorated function.
-    """
-    def wrapper(*args, **kwargs):
-        start = time.time()
-        result = func(*args, **kwargs)
-        end = time.time()
-        logger.info("Milvus {} run in {}s".format(func.__name__, round(end - start, 2)))
-        return result
-    return wrapper
-
-
-class MilvusClient(object):
-    def __init__(self, table_name=None, host=None, port=None):
-        self._milvus = Milvus()
-        self._table_name = table_name
-        try:
-            if not host:
-                self._milvus.connect(
-                    host = SERVER_HOST_DEFAULT,
-                    port = SERVER_PORT_DEFAULT)
-            else:
-                self._milvus.connect(
-                    host = host,
-                    port = port)
-        except Exception as e:
-            raise e
-
-    def __str__(self):
-        return 'Milvus table %s' % self._table_name
-
-    def check_status(self, status):
-        if not status.OK():
-            logger.error(status.message)
-            raise Exception("Status not ok")
-
-    def create_table(self, table_name, dimension, index_file_size, metric_type):
-        if not self._table_name:
-            self._table_name = table_name
-        if metric_type == "l2":
-            metric_type = MetricType.L2
-        elif metric_type == "ip":
-            metric_type = MetricType.IP
-        else:
-            logger.error("Not supported metric_type: %s" % metric_type)
-        self._metric_type = metric_type
-        create_param = {'table_name': table_name,
-                 'dimension': dimension,
-                 'index_file_size': index_file_size, 
-                 "metric_type": metric_type}
-        status = self._milvus.create_table(create_param)
-        self.check_status(status)
-
-    @time_wrapper
-    def insert(self, X, ids):
-        if self._metric_type == MetricType.IP:
-            logger.info("Set normalize for metric_type: Inner Product")
-            X = sklearn.preprocessing.normalize(X, axis=1, norm='l2')
-        X = X.astype(numpy.float32)
-        status, result = self._milvus.add_vectors(self._table_name, X.tolist(), ids=ids)
-        self.check_status(status)
-        return status, result
-
-    @time_wrapper
-    def create_index(self, index_type, nlist):
-        if index_type == "flat":
-            index_type = IndexType.FLAT
-        elif index_type == "ivf_flat":
-            index_type = IndexType.IVFLAT
-        elif index_type == "ivf_sq8":
-            index_type = IndexType.IVF_SQ8
-        elif index_type == "ivf_sq8h":
-            index_type = IndexType.IVF_SQ8H
-        elif index_type == "nsg":
-            index_type = IndexType.NSG
-        elif index_type == "ivf_pq":
-            index_type = IndexType.IVF_PQ
-        index_params = {
-            "index_type": index_type,
-            "nlist": nlist,
-        }
-        logger.info("Building index start, table_name: %s, index_params: %s" % (self._table_name, json.dumps(index_params)))
-        status = self._milvus.create_index(self._table_name, index=index_params, timeout=6*3600)
-        self.check_status(status)
-
-    def describe_index(self):
-        return self._milvus.describe_index(self._table_name)
-
-    def drop_index(self):
-        logger.info("Drop index: %s" % self._table_name)
-        return self._milvus.drop_index(self._table_name)
-
-    @time_wrapper
-    def query(self, X, top_k, nprobe):
-        if self._metric_type == MetricType.IP:
-            logger.info("Set normalize for metric_type: Inner Product")
-            X = sklearn.preprocessing.normalize(X, axis=1, norm='l2')
-        X = X.astype(numpy.float32)
-        status, results = self._milvus.search_vectors(self._table_name, top_k, nprobe, X.tolist())
-        self.check_status(status)
-        ids = []
-        for result in results:
-            tmp_ids = []
-            for item in result:
-                tmp_ids.append(item.id)
-            ids.append(tmp_ids)
-        return ids
-
-    def count(self):
-        return self._milvus.get_table_row_count(self._table_name)[1]
-
-    def delete(self, table_name):
-        logger.info("Start delete table: %s" % table_name)
-        return self._milvus.delete_table(table_name)
-
-    def describe(self):
-        return self._milvus.describe_table(self._table_name)
-
-    def exists_table(self, table_name):
-        return self._milvus.has_table(table_name)
-
-    def get_server_version(self):
-        status, res = self._milvus.server_version()
-        self.check_status(status)
-        return res
-
-    @time_wrapper
-    def preload_table(self):
-        return self._milvus.preload_table(self._table_name)

tests/milvus_ann_acc/config.yaml

@@ -1,17 +0,0 @@
-datasets: 
-  sift-128-euclidean: 
-    cpu_cache_size: 16
-    gpu_cache_size: 5
-    index_file_size: [1024]
-  nytimes-16-angular:
-    cpu_cache_size: 16
-    gpu_cache_size: 5
-    index_file_size: [1024]
-
-index:
-  index_types: ['flat', 'ivf_flat', 'ivf_sq8']
-  nlists: [8092, 16384]
-  
-search:
-  nprobes: [1, 8, 32]
-  top_ks: [10]

tests/milvus_ann_acc/main.py

@@ -1,57 +0,0 @@
-import os
-import sys
-import argparse
-from yaml import load, dump
-import logging
-from logging import handlers
-from client import MilvusClient
-import runner
-
-LOG_FOLDER = "logs"
-logger = logging.getLogger("milvus_acc")
-formatter = logging.Formatter('[%(asctime)s] [%(levelname)-4s] [%(pathname)s:%(lineno)d] %(message)s')
-if not os.path.exists(LOG_FOLDER):
-    os.system('mkdir -p %s' % LOG_FOLDER)
-fileTimeHandler = handlers.TimedRotatingFileHandler(os.path.join(LOG_FOLDER, 'acc'), "D", 1, 10)
-fileTimeHandler.suffix = "%Y%m%d.log"
-fileTimeHandler.setFormatter(formatter)
-logging.basicConfig(level=logging.DEBUG)
-fileTimeHandler.setFormatter(formatter)
-logger.addHandler(fileTimeHandler)
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser.add_argument(
-        "--host",
-        default="127.0.0.1",
-        help="server host")
-    parser.add_argument(
-        "--port",
-        default=19530,
-        help="server port")   
-    parser.add_argument(
-        '--suite',
-        metavar='FILE',
-        help='load config definitions from suite_czr'
-             '.yaml',
-        default='suite_czr.yaml')
-    args = parser.parse_args()
-    if args.suite:
-        with open(args.suite, "r") as f:
-            suite = load(f)
-            hdf5_path = suite["hdf5_path"]
-            dataset_configs = suite["datasets"]
-            if not hdf5_path or not dataset_configs:
-                logger.warning("No datasets given")
-                sys.exit()
-            f.close()
-    for dataset_config in dataset_configs:
-        logger.debug(dataset_config)
-        milvus_instance = MilvusClient(host=args.host, port=args.port)
-        runner.run(milvus_instance, dataset_config, hdf5_path)
-
-
-if __name__ == "__main__":
-    main()

tests/milvus_ann_acc/requirements.txt

@@ -1,9 +0,0 @@
-numpy==1.16.3
-pymilvus-test>=0.2.0
-scikit-learn==0.19.1
-h5py==2.7.1
-influxdb==5.2.2
-pyyaml>=5.1
-tableprint==0.8.0
-ansicolors==1.1.8
-scipy==1.3.1

tests/milvus_ann_acc/runner.py

@@ -1,162 +0,0 @@
-import os
-import pdb
-import time
-import random
-import sys
-import logging
-import h5py
-import numpy
-from influxdb import InfluxDBClient
-
-INSERT_INTERVAL = 100000
-# s
-DELETE_INTERVAL_TIME = 5
-INFLUXDB_HOST = "192.168.1.194"
-INFLUXDB_PORT = 8086
-INFLUXDB_USER = "admin"
-INFLUXDB_PASSWD = "admin"
-INFLUXDB_NAME = "test_result"
-influxdb_client = InfluxDBClient(host=INFLUXDB_HOST, port=INFLUXDB_PORT, username=INFLUXDB_USER, password=INFLUXDB_PASSWD, database=INFLUXDB_NAME)
-
-logger = logging.getLogger("milvus_acc.runner")
-
-
-def parse_dataset_name(dataset_name):
-    data_type = dataset_name.split("-")[0]
-    dimension = int(dataset_name.split("-")[1])
-    metric = dataset_name.split("-")[-1]
-    # metric = dataset.attrs['distance']
-    # dimension = len(dataset["train"][0])
-    if metric == "euclidean":
-        metric_type = "l2"
-    elif metric  == "angular":
-        metric_type = "ip"
-    return ("ann"+data_type, dimension, metric_type)
-
-
-def get_dataset(hdf5_path, dataset_name):
-    file_path = os.path.join(hdf5_path, '%s.hdf5' % dataset_name)
-    if not os.path.exists(file_path):
-        raise Exception("%s not existed" % file_path)
-    dataset = h5py.File(file_path)
-    return dataset
-
-
-def get_table_name(hdf5_path, dataset_name, index_file_size):
-    data_type, dimension, metric_type = parse_dataset_name(dataset_name)
-    dataset = get_dataset(hdf5_path, dataset_name)
-    table_size = len(dataset["train"])
-    table_size = str(table_size // 1000000)+"m"
-    table_name = data_type+'_'+table_size+'_'+str(index_file_size)+'_'+str(dimension)+'_'+metric_type
-    return table_name
-
-
-def recall_calc(result_ids, true_ids, top_k, recall_k):
-    sum_intersect_num = 0
-    recall = 0.0
-    for index, result_item in enumerate(result_ids):
-        # if len(set(true_ids[index][:top_k])) != len(set(result_item)):
-        #     logger.warning("Error happened: query result length is wrong")
-        #     continue
-        tmp = set(true_ids[index][:recall_k]).intersection(set(result_item))
-        sum_intersect_num = sum_intersect_num + len(tmp)
-    recall = round(sum_intersect_num / (len(result_ids) * recall_k), 4)
-    return recall
-
-
-def run(milvus, config, hdf5_path, force=True):
-    server_version = milvus.get_server_version()
-    logger.info(server_version)
-    
-    for dataset_name, config_value in config.items():
-        dataset = get_dataset(hdf5_path, dataset_name)
-        index_file_sizes = config_value["index_file_sizes"]
-        index_types = config_value["index_types"]
-        nlists = config_value["nlists"]
-        search_param = config_value["search_param"]
-        top_ks = search_param["top_ks"]
-        nprobes = search_param["nprobes"]
-        nqs = search_param["nqs"]
-
-        for index_file_size in index_file_sizes:
-            table_name = get_table_name(hdf5_path, dataset_name, index_file_size)  
-            if milvus.exists_table(table_name):
-                if force is True:
-                    logger.info("Re-create table: %s" % table_name)
-                    milvus.delete(table_name)
-                    time.sleep(DELETE_INTERVAL_TIME)
-                else:
-                    logger.warning("Table name: %s existed" % table_name)
-                    continue
-            data_type, dimension, metric_type = parse_dataset_name(dataset_name)
-            milvus.create_table(table_name, dimension, index_file_size, metric_type)
-            logger.info(milvus.describe())
-            insert_vectors = numpy.array(dataset["train"])
-            # milvus.insert(insert_vectors)
-
-            loops = len(insert_vectors) // INSERT_INTERVAL + 1
-            for i in range(loops):
-                start = i*INSERT_INTERVAL
-                end = min((i+1)*INSERT_INTERVAL, len(insert_vectors))
-                tmp_vectors = insert_vectors[start:end]
-                if start < end:
-                    milvus.insert(tmp_vectors, ids=[i for i in range(start, end)])
-            time.sleep(20)
-            row_count = milvus.count()
-            logger.info("Table: %s, row count: %s" % (table_name, row_count))
-            if milvus.count() != len(insert_vectors):
-                logger.error("Table row count is not equal to insert vectors")
-                return
-            for index_type in index_types:
-                for nlist in nlists:
-                    milvus.create_index(index_type, nlist)
-                    logger.info(milvus.describe_index())
-                    logger.info("Start preload table: %s, index_type: %s, nlist: %s" % (table_name, index_type, nlist))
-                    milvus.preload_table()
-                    true_ids = numpy.array(dataset["neighbors"])
-                    for nprobe in nprobes:
-                        for nq in nqs:
-                            query_vectors = numpy.array(dataset["test"][:nq])
-                            for top_k in top_ks:
-                                rec1 = 0.0
-                                rec10 = 0.0
-                                rec100 = 0.0
-                                result_ids = milvus.query(query_vectors, top_k, nprobe)
-                                logger.info("Query result: %s" % len(result_ids))
-                                rec1 = recall_calc(result_ids, true_ids, top_k, 1)
-                                if top_k == 10:
-                                    rec10 = recall_calc(result_ids, true_ids, top_k, 10)
-                                if top_k == 100:
-                                    rec10 = recall_calc(result_ids, true_ids, top_k, 10)
-                                    rec100 = recall_calc(result_ids, true_ids, top_k, 100)
-                                avg_radio = recall_calc(result_ids, true_ids, top_k, top_k)
-                                logger.debug("Recall_1: %s" % rec1)
-                                logger.debug("Recall_10: %s" % rec10)
-                                logger.debug("Recall_100: %s" % rec100)
-                                logger.debug("Accuracy: %s" % avg_radio)
-                                acc_record = [{
-                                    "measurement": "accuracy",
-                                    "tags": {
-                                        "server_version": server_version,
-                                        "dataset": dataset_name,
-                                        "index_file_size": index_file_size,
-                                        "index_type": index_type,
-                                        "nlist": nlist,
-                                        "search_nprobe": nprobe,
-                                        "top_k": top_k,
-                                        "nq": len(query_vectors)
-                                    },
-                                    # "time": time.ctime(),
-                                    "time": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
-                                    "fields": {
-                                        "recall1": rec1,
-                                        "recall10": rec10,
-                                        "recall100": rec100,
-                                        "avg_radio": avg_radio
-                                    }
-                                }]
-                                logger.info(acc_record)
-                                try:
-                                    res = influxdb_client.write_points(acc_record)
-                                except Exception as e:
-                                    logger.error("Insert infuxdb failed: %s" % str(e))

tests/milvus_ann_acc/suite.yaml

@@ -1,29 +0,0 @@
-datasets:
-  - sift-128-euclidean: 
-      index_file_sizes: [1024]
-      index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h']
-      # index_types: ['ivf_sq8']
-      nlists: [16384]
-      search_param:
-        nprobes: [1, 8, 16, 32, 64, 128, 256, 512]
-        top_ks: [10]
-        nqs: [10000]
-  - glove-25-angular:
-      index_file_sizes: [1024]
-      index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h']
-      # index_types: ['ivf_sq8']
-      nlists: [16384]
-      search_param:
-        nprobes: [1, 8, 16, 32, 64, 128, 256, 512]
-        top_ks: [10]
-        nqs: [10000]
-  - glove-200-angular:
-      index_file_sizes: [1024]
-      index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h']
-      # index_types: ['ivf_sq8']
-      nlists: [16384]
-      search_param:
-        nprobes: [1, 8, 16, 32, 64, 128, 256, 512]
-        top_ks: [10]
-        nqs: [10000]
-hdf5_path: /test/milvus/ann_hdf5/

tests/milvus_ann_acc/suite.yaml.bak

@@ -1,11 +0,0 @@
-datasets:
-  - glove-200-angular:
-      index_file_sizes: [1024]
-      index_types: ['ivf_sq8']
-      # index_types: ['ivf_sq8']
-      nlists: [16384]
-      search_param:
-        nprobes: [256, 400, 256]
-        top_ks: [100]
-        nqs: [10000]
-hdf5_path: /test/milvus/ann_hdf5/

tests/milvus_ann_acc/suite_cpu.yaml

@@ -1,19 +0,0 @@
-datasets:
-  - sift-128-euclidean: 
-      index_file_sizes: [1024]
-      index_types: ['ivf_flat', 'ivf_sq8']
-      nlists: [16384]
-      search_param:
-        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
-        top_ks: [10]
-        nqs: [10000]
-  - glove-200-angular:
-      index_file_sizes: [1024]
-      index_types: ['ivf_flat', 'ivf_sq8']
-      # index_types: ['ivf_sq8']
-      nlists: [16384]
-      search_param:
-        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
-        top_ks: [10]
-        nqs: [10000]
-hdf5_path: /test/milvus/ann_hdf5/

tests/milvus_ann_acc/suite_czr.yaml

@@ -1,20 +0,0 @@
-datasets:
-  - sift-128-euclidean:
-      index_file_sizes: [1024]
-      index_types: ['ivf_sq8', 'ivf_sq8h']
-      # index_types: ['ivf_sq8']
-      nlists: [16384]
-      search_param:
-        nprobes: [16, 128, 1024]
-        top_ks: [1, 10, 100]
-        nqs: [10, 100, 1000]
-  - glove-200-angular:
-      index_file_sizes: [1024]
-      index_types: ['ivf_sq8', 'ivf_sq8h']
-      # index_types: ['ivf_sq8']
-      nlists: [16384]
-      search_param:
-        nprobes: [16, 128, 1024]
-        top_ks: [1, 10, 100]
-        nqs: [10, 100, 1000]
-hdf5_path: /test/milvus/ann_hdf5/

tests/milvus_ann_acc/suite_debug.yaml

@@ -1,19 +0,0 @@
-datasets:
-  - sift-128-euclidean: 
-      index_file_sizes: [1024]
-      index_types: ['ivf_flat', 'ivf_sq8']
-      nlists: [16384]
-      search_param:
-        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
-        top_ks: [10]
-        nqs: [10000]
-  - glove-200-angular:
-      index_file_sizes: [1024]
-      index_types: ['ivf_flat', 'ivf_sq8']
-      # index_types: ['ivf_sq8']
-      nlists: [16384]
-      search_param:
-        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
-        top_ks: [10]
-        nqs: [10000]
-hdf5_path: /test/milvus/ann_hdf5/

tests/milvus_ann_acc/suite_gpu.yaml

@@ -1,19 +0,0 @@
-datasets:
-  - sift-128-euclidean: 
-      index_file_sizes: [1024]
-      index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h', 'ivf_pq']
-      nlists: [16384]
-      search_param:
-        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
-        top_ks: [10]
-        nqs: [10000]
-  - glove-200-angular:
-      index_file_sizes: [1024]
-      index_types: ['ivf_flat', 'ivf_sq8']
-      # index_types: ['ivf_sq8']
-      nlists: [16384]
-      search_param:
-        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
-        top_ks: [10]
-        nqs: [10000]
-hdf5_path: /test/milvus/ann_hdf5/

tests/milvus_ann_acc/test.py

@@ -1,33 +0,0 @@
-import time
-from influxdb import InfluxDBClient
-
-INFLUXDB_HOST = "192.168.1.194"
-INFLUXDB_PORT = 8086
-INFLUXDB_USER = "admin"
-INFLUXDB_PASSWD = "admin"
-INFLUXDB_NAME = "test_result"
-
-client = InfluxDBClient(host=INFLUXDB_HOST, port=INFLUXDB_PORT, username=INFLUXDB_USER, password=INFLUXDB_PASSWD, database=INFLUXDB_NAME)
-
-print(client.get_list_database())
-acc_record = [{
-    "measurement": "accuracy",
-    "tags": {
-        "server_version": "0.4.3",
-        "dataset": "test",
-        "index_type": "test",
-        "nlist": 12,
-        "search_nprobe": 12,
-        "top_k": 1,
-        "nq": 1
-    },
-    "time": time.ctime(),
-    "fields": {
-        "accuracy": 0.1
-    }
-}]
-try:
-	res = client.write_points(acc_record)
-	print(res)
-except Exception as e:
-    print(str(e))

tests/milvus_benchmark/README.md

@@ -1,23 +1,42 @@
-# Requirements
+# Quick start
 
-- python 3.6+
-- pip install -r requirements.txt
+## 运行
 
-# How to use this Test Project
+### 运行说明：
 
-This project is used to test performance / accuracy /  stability of milvus server
+- 用于进行大数据集的准确性、性能、以及稳定性等相关测试
+- 可以运行两种模式：基于K8S+Jenkins的测试模式，以及local模式
 
-1. update your test configuration in suites_*.yaml
-2. run command
+### 运行示例：
 
-```shell
-### docker mode:
-python main.py --image=milvusdb/milvus:latest --run-count=2 --run-type=performance
+1. 基于K8S+Jenkins的测试方式：
 
-### local mode:
-python main.py --local --run-count=2 --run-type=performance --ip=127.0.0.1 --port=19530
-```
+   ![](assets/Parameters.png)
 
-# Contribution getting started
+2. 本地测试：
 
-- Follow PEP-8 for naming and black for formatting.
+   `python3 main.py --local --host=*.* --port=19530 --suite=suites/gpu_search_performance_random50m.yaml`
+
+### 测试集配置文件：
+
+在进行自定义的测试集或测试参数时，需要编写测试集配置文件。
+
+下面是搜索性能的测试集配置文件：
+
+![](assets/gpu_search_performance_random50m-yaml.png)
+
+1. search_performance: 定义测试类型，还包括`build_performance`,`insert_performance`,`accuracy`,`stability`,`search_stability`
+2. tables: 定义测试集列表
+3. 对于`search_performance`这类测试，每个table下都包含：
+   - server: milvus的server_config
+   - table_name: 表名，当前框架仅支持单表操作
+   - run_count: 搜索运行次数，并取搜索时间的最小值作为指标
+   - search_params: 向量搜索参数
+
+## 测试结果：
+
+搜索性能的结果输出：
+
+![](assets/milvus-nightly-performance-new-jenkins.png)
+
+在K8S测试模式下时，除打印上面的输出外，还会进行数据上报

tests/milvus_benchmark/ci/jenkinsfile/cleanup.groovy

@@ -0,0 +1,13 @@
+try {
+    def result = sh script: "helm status benchmark-test-${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true
+    if (!result) {
+        sh "helm del --purge benchmark-test-${env.JOB_NAME}-${env.BUILD_NUMBER}"
+    }
+} catch (exc) {
+    def result = sh script: "helm status benchmark-test-${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true
+    if (!result) {
+        sh "helm del --purge benchmark-test-${env.JOB_NAME}-${env.BUILD_NUMBER}"
+    }
+    throw exc
+}
+

tests/milvus_benchmark/ci/jenkinsfile/deploy_test.groovy

@@ -0,0 +1,20 @@
+timeout(time: 1440, unit: 'MINUTES') {
+    try {
+        dir ("milvus-helm") {
+            sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
+            sh 'helm repo update'
+            checkout([$class: 'GitSCM', branches: [[name: "${HELM_BRANCH}"]], userRemoteConfigs: [[url: "${HELM_URL}", name: 'origin', refspec: "+refs/heads/${HELM_BRANCH}:refs/remotes/origin/${HELM_BRANCH}"]]])
+        }
+        dir ("milvus_benchmark") {
+            print "Git clone url: ${TEST_URL}:${TEST_BRANCH}"
+            checkout([$class: 'GitSCM', branches: [[name: "${TEST_BRANCH}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "${TEST_URL}", name: 'origin', refspec: "+refs/heads/${TEST_BRANCH}:refs/remotes/origin/${TEST_BRANCH}"]]])
+            print "Install requirements"
+            sh "python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com"
+            sh "python3 -m pip install git+${TEST_LIB_URL}"
+            sh "python3 main.py --hostname=${params.SERVER_HOST} --image-tag=${IMAGE_TAG} --image-type=${params.IMAGE_TYPE} --suite=suites/${params.SUITE}"
+        }
+    } catch (exc) {
+        echo 'Deploy Test Failed !'
+        throw exc
+    }
+}

tests/milvus_benchmark/ci/main_jenkinsfile

@@ -6,9 +6,10 @@ pipeline {
     }
 
     parameters{
-        choice choices: ['cpu', 'gpu'], description: 'cpu or gpu version', name: 'IMAGE_TYPE'
-        string defaultValue: '0.6.0', description: 'server image version', name: 'IMAGE_VERSION', trim: true
-        string defaultValue: 'suite.yaml', description: 'test suite config yaml', name: 'SUITE', trim: true
+        choice choices: ['gpu', 'cpu'], description: 'cpu or gpu version', name: 'IMAGE_TYPE'
+        string defaultValue: 'master', description: 'server image version', name: 'IMAGE_VERSION', trim: true
+        choice choices: ['poseidon', 'eros', 'apollo', 'athena'], description: 'server host', name: 'SERVER_HOST'
+        string defaultValue: 'gpu_search_performance_sift1m.yaml', description: 'test suite config yaml', name: 'SUITE', trim: true
         string defaultValue: '09509e53-9125-4f5d-9ce8-42855987ad67', description: 'git credentials', name: 'GIT_USER', trim: true
     }
 
@@ -16,15 +17,16 @@ pipeline {
         IMAGE_TAG = "${params.IMAGE_VERSION}-${params.IMAGE_TYPE}-ubuntu18.04-release"
         HELM_URL = "https://github.com/milvus-io/milvus-helm.git"
         HELM_BRANCH = "0.6.0"
-        TEST_URL = "git@192.168.1.105:Test/milvus_ann_acc.git"
-        TEST_BRANCH = "0.6.0"
+        TEST_URL = "git@192.168.1.105:Test/milvus_benchmark.git"
+        TEST_BRANCH = "master"
+        TEST_LIB_URL = "http://192.168.1.105:6060/Test/milvus_metrics.git"
     }
 
     stages {
         stage("Setup env") {
             agent {
                 kubernetes {
-                    label 'dev-test'
+                    label "test-benchmark-${env.JOB_NAME}-${env.BUILD_NUMBER}"
                     defaultContainer 'jnlp'
                     yaml """
                         apiVersion: v1
@@ -44,14 +46,26 @@ pipeline {
                             - name: kubeconf
                               mountPath: /root/.kube/
                               readOnly: true
-                            - name: hdf5-path
-                              mountPath: /test
+                            - name: raw-data-path
+                              mountPath: /poc
                               readOnly: true
+                            - name: db-data-path
+                              mountPath: /test
+                              readOnly: false
                           volumes:
                           - name: kubeconf
                             secret:
                               secretName: test-cluster-config
-                          - name: hdf5-path
+                          - name: raw-data-path
+                            flexVolume:
+                              driver: "fstab/cifs"
+                              fsType: "cifs"
+                              secretRef:
+                                name: "cifs-test-secret"
+                              options:
+                                networkPath: "//192.168.1.126/poc"
+                                mountOptions: "vers=1.0"
+                          - name: db-data-path
                             flexVolume:
                               driver: "fstab/cifs"
                               fsType: "cifs"
@@ -65,30 +79,19 @@ pipeline {
             }
 
             stages {
-                stage("Deploy Default Server") {
+                stage("Deploy Test") {
                     steps {
-                        gitlabCommitStatus(name: 'Accuracy Test') {
+                        gitlabCommitStatus(name: 'Deploy Test') {
                             container('milvus-testframework') {
                                 script {
-                                    print "In Deploy Default Server Stage"
-                                    load "${env.WORKSPACE}/ci/jenkinsfile/deploy_default_server.groovy"
-                                }
-                            }
-                        }
-                    }
-                }
-                stage("Acc Test") {
-                    steps {
-                        gitlabCommitStatus(name: 'Accuracy Test') {
-                            container('milvus-testframework') {
-                                script {
-                                    print "In Acc test stage"
-                                    load "${env.WORKSPACE}/ci/jenkinsfile/acc_test.groovy"
+                                    print "In Deploy Test Stage"
+                                    load "${env.WORKSPACE}/ci/jenkinsfile/deploy_test.groovy"
                                 }
                             }
                         }
                     }
                 }
+
                 stage ("Cleanup Env") {
                     steps {
                         gitlabCommitStatus(name: 'Cleanup Env') {
@@ -111,17 +114,17 @@ pipeline {
                 }
                 success {
                     script {
-                        echo "Milvus ann-accuracy test success !"
+                        echo "Milvus benchmark test success !"
                     }
                 }
                 aborted {
                     script {
-                        echo "Milvus ann-accuracy test aborted !"
+                        echo "Milvus benchmark test aborted !"
                     }
                 }
                 failure {
                     script {
-                        echo "Milvus ann-accuracy test failed !"
+                        echo "Milvus benchmark test failed !"
                     }
                 }
             }

tests/milvus_benchmark/client.py

@@ -1,8 +1,8 @@
+import sys
 import pdb
 import random
 import logging
 import json
-import sys
 import time, datetime
 from multiprocessing import Process
 from milvus import Milvus, IndexType, MetricType
@@ -12,7 +12,14 @@ logger = logging.getLogger("milvus_benchmark.client")
 SERVER_HOST_DEFAULT = "127.0.0.1"
 # SERVER_HOST_DEFAULT = "192.168.1.130"
 SERVER_PORT_DEFAULT = 19530
-
+INDEX_MAP = {
+    "flat": IndexType.FLAT,
+    "ivf_flat": IndexType.IVFLAT,
+    "ivf_sq8": IndexType.IVF_SQ8,
+    "nsg": IndexType.RNSG,
+    "ivf_sq8h": IndexType.IVF_SQ8H,
+    "ivf_pq": IndexType.IVF_PQ
+}
 
 def time_wrapper(func):
     """
@@ -28,18 +35,30 @@ def time_wrapper(func):
 
 
 class MilvusClient(object):
-    def __init__(self, table_name=None, ip=None, port=None):
+    def __init__(self, table_name=None, ip=None, port=None, timeout=60):
         self._milvus = Milvus()
         self._table_name = table_name
         try:
+            i = 1
+            start_time = time.time()
             if not ip:
                 self._milvus.connect(
                     host = SERVER_HOST_DEFAULT,
                     port = SERVER_PORT_DEFAULT)
             else:
-                self._milvus.connect(
-                    host = ip,
-                    port = port)
+                # retry connect for remote server
+                while time.time() < start_time + timeout:
+                    try:
+                        self._milvus.connect(
+                            host = ip,
+                            port = port)
+                        if self._milvus.connected() is True:
+                            logger.debug("Try connect times: %d, %s" % (i, round(time.time() - start_time, 2)))
+                            break
+                    except Exception as e:
+                        logger.debug("Milvus connect failed")
+                        i = i + 1
+
         except Exception as e:
             raise e
 
@@ -49,7 +68,7 @@ class MilvusClient(object):
     def check_status(self, status):
         if not status.OK():
             logger.error(status.message)
-            raise Exception("Status not ok")
+            # raise Exception("Status not ok")
 
     def create_table(self, table_name, dimension, index_file_size, metric_type):
         if not self._table_name:
@@ -58,6 +77,10 @@ class MilvusClient(object):
             metric_type = MetricType.L2
         elif metric_type == "ip":
             metric_type = MetricType.IP
+        elif metric_type == "jaccard":
+            metric_type = MetricType.JACCARD
+        elif metric_type == "hamming":
+            metric_type = MetricType.HAMMING
         else:
             logger.error("Not supported metric_type: %s" % metric_type)
         create_param = {'table_name': table_name,
@@ -75,20 +98,8 @@ class MilvusClient(object):
 
     @time_wrapper
     def create_index(self, index_type, nlist):
-        if index_type == "flat":
-            index_type = IndexType.FLAT
-        elif index_type == "ivf_flat":
-            index_type = IndexType.IVFLAT
-        elif index_type == "ivf_sq8":
-            index_type = IndexType.IVF_SQ8
-        elif index_type == "nsg":
-            index_type = IndexType.NSG
-        elif index_type == "ivf_sq8h":
-            index_type = IndexType.IVF_SQ8H
-        elif index_type == "ivf_pq":
-            index_type = IndexType.IVF_PQ
         index_params = {
-            "index_type": index_type,
+            "index_type": INDEX_MAP[index_type],
             "nlist": nlist,
         }
         logger.info("Building index start, table_name: %s, index_params: %s" % (self._table_name, json.dumps(index_params)))
@@ -96,7 +107,18 @@ class MilvusClient(object):
         self.check_status(status)
 
     def describe_index(self):
-        return self._milvus.describe_index(self._table_name)
+        status, result = self._milvus.describe_index(self._table_name)
+        index_type = None
+        for k, v in INDEX_MAP.items():
+            if result._index_type == v:
+                index_type = k
+                break
+        nlist = result._nlist
+        res = {
+            "index_type": index_type,
+            "nlist": nlist
+        }
+        return res
 
     def drop_index(self):
         logger.info("Drop index: %s" % self._table_name)
@@ -106,7 +128,7 @@ class MilvusClient(object):
     def query(self, X, top_k, nprobe):
         status, result = self._milvus.search_vectors(self._table_name, top_k, nprobe, X)
         self.check_status(status)
-        return status, result
+        return result
 
     def count(self):
         return self._milvus.get_table_row_count(self._table_name)[1]
@@ -122,7 +144,7 @@ class MilvusClient(object):
                 continue
             else:
                 break
-        if i < timeout:
+        if i >= timeout:
             logger.error("Delete table timeout")
 
     def describe(self):
@@ -131,8 +153,10 @@ class MilvusClient(object):
     def show_tables(self):
         return self._milvus.show_tables()
 
-    def exists_table(self):
-        status, res = self._milvus.has_table(self._table_name)
+    def exists_table(self, table_name=None):
+        if table_name is None:
+            table_name = self._table_name
+        status, res = self._milvus.has_table(table_name)
         self.check_status(status)
         return res
 
@@ -140,6 +164,33 @@ class MilvusClient(object):
     def preload_table(self):
         return self._milvus.preload_table(self._table_name, timeout=3000)
 
+    def get_server_version(self):
+        status, res = self._milvus.server_version()
+        return res
+
+    def get_server_mode(self):
+        return self.cmd("mode")
+
+    def get_server_commit(self):
+        return self.cmd("build_commit_id")
+
+    def get_server_config(self):
+        return json.loads(self.cmd("get_config *"))
+
+    def get_mem_info(self):
+        result = json.loads(self.cmd("get_system_info"))
+        result_human = {
+            # unit: Gb
+            "memory_used": round(int(result["memory_used"]) / (1024*1024*1024), 2)
+        }
+        return result_human
+
+    def cmd(self, command):
+        status, res = self._milvus._cmd(command)
+        logger.info("Server command: %s, result: %s" % (command, res))
+        self.check_status(status)
+        return res
+
 
 def fit(table_name, X):
     milvus = Milvus()
@@ -265,6 +316,12 @@ if __name__ == "__main__":
     # data = mmap_fvecs("/poc/deep1b/deep1B_queries.fvecs")
     # data = sklearn.preprocessing.normalize(data, axis=1, norm='l2')
     # np.save("/test/milvus/deep1b/query.npy", data)
+    dimension = 4096
+    insert_xb = 10000
+    insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)]
+    data = sklearn.preprocessing.normalize(insert_vectors, axis=1, norm='l2')
+    np.save("/test/milvus/raw_data/random/query_%d.npy" % dimension, data)
+    sys.exit()
 
     total_size = 100000000
     # total_size = 1000000000

tests/milvus_benchmark/demo.py

@@ -1,51 +0,0 @@
-import os
-import logging
-import pdb
-import time
-import random
-from multiprocessing import Process
-import numpy as np
-from client import MilvusClient
-
-nq = 100000
-dimension = 128
-run_count = 1
-table_name = "sift_10m_1024_128_ip"
-insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)]
-
-def do_query(milvus, table_name, top_ks, nqs, nprobe, run_count):
-    bi_res = []
-    for index, nq in enumerate(nqs):
-        tmp_res = []
-        for top_k in top_ks:
-            avg_query_time = 0.0
-            total_query_time = 0.0
-            vectors = insert_vectors[0:nq]
-            for i in range(run_count):
-                start_time = time.time()
-                status, query_res = milvus.query(vectors, top_k, nprobe)
-                total_query_time = total_query_time + (time.time() - start_time)
-                if status.code:
-                    print(status.message)
-            avg_query_time = round(total_query_time / run_count, 2)
-            tmp_res.append(avg_query_time)
-        bi_res.append(tmp_res)
-    return bi_res
-
-while 1:
-    milvus_instance = MilvusClient(table_name, ip="192.168.1.197", port=19530)
-    top_ks = random.sample([x for x in range(1, 100)], 4)
-    nqs = random.sample([x for x in range(1, 1000)], 3)
-    nprobe = random.choice([x for x in range(1, 500)])
-    res = do_query(milvus_instance, table_name, top_ks, nqs, nprobe, run_count)
-    status, res = milvus_instance.insert(insert_vectors, ids=[x for x in range(len(insert_vectors))])
-    if not status.OK():
-        logger.error(status.message)
-
-    # status = milvus_instance.drop_index()
-    if not status.OK():
-        print(status.message)
-    index_type = "ivf_sq8"
-    status = milvus_instance.create_index(index_type, 16384)
-    if not status.OK():
-        print(status.message)

tests/milvus_benchmark/docker_runner.py

@@ -53,7 +53,6 @@ class DockerRunner(Runner):
                         # milvus.create_index("ivf_sq8", 16384)
                         res = self.do_insert(milvus, table_name, data_type, dimension, table_size, param["ni_per"])
                         logger.info(res)
-
                         # wait for file merge
                         time.sleep(table_size * dimension / 5000000)
                         # Clear up
@@ -71,7 +70,7 @@ class DockerRunner(Runner):
                         container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None)
                         time.sleep(2)
                         milvus = MilvusClient(table_name)
-                        logger.debug(milvus._milvus.show_tables())
+                        logger.debug(milvus.show_tables())
                         # Check has table or not
                         if not milvus.exists_table():
                             logger.warning("Table %s not existed, continue exec next params ..." % table_name)
@@ -104,6 +103,92 @@ class DockerRunner(Runner):
                                     utils.print_table(headers, nqs, res)
                         utils.remove_container(container)
 
+        elif run_type == "insert_performance":
+            for op_type, op_value in definition.items():
+                # run docker mode
+                run_count = op_value["run_count"]
+                run_params = op_value["params"]
+                container = None
+                if not run_params:
+                    logger.debug("No run params")
+                    continue
+                for index, param in enumerate(run_params):
+                    logger.info("Definition param: %s" % str(param))
+                    table_name = param["table_name"]
+                    volume_name = param["db_path_prefix"]
+                    print(table_name)
+                    (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
+                    for k, v in param.items():
+                        if k.startswith("server."):
+                            # Update server config
+                            utils.modify_config(k, v, type="server", db_slave=None)
+                    container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None)
+                    time.sleep(2)
+                    milvus = MilvusClient(table_name)
+                    # Check has table or not
+                    if milvus.exists_table():
+                        milvus.delete()
+                        time.sleep(10)
+                    milvus.create_table(table_name, dimension, index_file_size, metric_type)
+                    # debug
+                    # milvus.create_index("ivf_sq8", 16384)
+                    res = self.do_insert(milvus, table_name, data_type, dimension, table_size, param["ni_per"])
+                    logger.info(res)
+                    # wait for file merge
+                    time.sleep(table_size * dimension / 5000000)
+                    # Clear up
+                    utils.remove_container(container)
+
+        elif run_type == "search_performance":
+            for op_type, op_value in definition.items():
+                # run docker mode
+                run_count = op_value["run_count"]
+                run_params = op_value["params"]
+                container = None
+                for index, param in enumerate(run_params):
+                    logger.info("Definition param: %s" % str(param))
+                    table_name = param["dataset"]
+                    volume_name = param["db_path_prefix"]
+                    (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
+                    for k, v in param.items():
+                        if k.startswith("server."):                   
+                            utils.modify_config(k, v, type="server")
+                    container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None)
+                    time.sleep(2)
+                    milvus = MilvusClient(table_name)
+                    logger.debug(milvus.show_tables())
+                    # Check has table or not
+                    if not milvus.exists_table():
+                        logger.warning("Table %s not existed, continue exec next params ..." % table_name)
+                        continue
+                    # parse index info
+                    index_types = param["index.index_types"]
+                    nlists = param["index.nlists"]
+                    # parse top-k, nq, nprobe
+                    top_ks, nqs, nprobes = parser.search_params_parser(param)
+                    for index_type in index_types:
+                        for nlist in nlists:
+                            result = milvus.describe_index()
+                            logger.info(result)
+                            # milvus.drop_index()
+                            # milvus.create_index(index_type, nlist)
+                            result = milvus.describe_index()
+                            logger.info(result)
+                            logger.info(milvus.count())
+                            # preload index
+                            milvus.preload_table()
+                            logger.info("Start warm up query")
+                            res = self.do_query(milvus, table_name, [1], [1], 1, 1)
+                            logger.info("End warm up query")
+                            # Run query test
+                            for nprobe in nprobes:
+                                logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe))
+                                res = self.do_query(milvus, table_name, top_ks, nqs, nprobe, run_count)
+                                headers = ["Nq/Top-k"]
+                                headers.extend([str(top_k) for top_k in top_ks])
+                                utils.print_table(headers, nqs, res)
+                    utils.remove_container(container)
+
         elif run_type == "accuracy":
             """
             {
@@ -149,11 +234,9 @@ class DockerRunner(Runner):
                     nlists = param["index.nlists"]
                     # parse top-k, nq, nprobe
                     top_ks, nqs, nprobes = parser.search_params_parser(param)
-
                     if sift_acc is True:
                         # preload groundtruth data
                         true_ids_all = self.get_groundtruth_ids(table_size)
-
                     acc_dict = {}
                     for index_type in index_types:
                         for nlist in nlists:

tests/milvus_benchmark/k8s_runner.py

@@ -0,0 +1,473 @@
+import os
+import logging
+import pdb
+import time
+import re
+import random
+import traceback
+from multiprocessing import Process
+import numpy as np
+from client import MilvusClient
+import utils
+import parser
+from runner import Runner
+from milvus_metrics.api import report
+from milvus_metrics.models import Env, Hardware, Server, Metric
+import utils
+
+logger = logging.getLogger("milvus_benchmark.k8s_runner")
+namespace = "milvus"
+DELETE_INTERVAL_TIME = 5
+# INSERT_INTERVAL = 100000
+INSERT_INTERVAL = 50000
+timestamp = int(time.time())
+default_path = "/var/lib/milvus"
+
+class K8sRunner(Runner):
+    """run docker mode"""
+    def __init__(self):
+        super(K8sRunner, self).__init__()
+        self.name = utils.get_unique_name()
+        self.host = None
+        self.ip = None
+        self.hostname = None
+        self.env_value = None
+        
+    def init_env(self, server_config, args):
+        self.hostname = args.hostname
+        # update server_config
+        helm_path = os.path.join(os.getcwd(), "../milvus-helm/milvus")
+        server_config_file = helm_path+"/ci/config/sqlite/%s/server_config.yaml" % (args.image_type)
+        if not os.path.exists(server_config_file):
+            raise Exception("File %s not existed" % server_config_file)
+        if server_config:
+            logger.debug("Update server config")
+            utils.update_server_config(server_config_file, server_config)
+        # update log_config
+        log_config_file = helm_path+"/config/log_config.conf"
+        if not os.path.exists(log_config_file):
+            raise Exception("File %s not existed" % log_config_file)
+        src_log_config_file = helm_path+"/config/log_config.conf.src"
+        if not os.path.exists(src_log_config_file):
+            # copy
+            os.system("cp %s %s" % (log_config_file, src_log_config_file))
+        else:
+            # reset
+            os.system("cp %s %s" % (src_log_config_file, log_config_file))
+        if "db_config.primary_path" in server_config:
+            os.system("sed -i 's#%s#%s#g' %s" % (default_path, server_config["db_config.primary_path"], log_config_file))
+        
+        # with open(log_config_file, "r+") as fd:
+        #     for line in fd.readlines():
+        #         fd.write(re.sub(r'^%s' % default_path, server_config["db_config.primary_path"], line))
+        # update values
+        values_file_path = helm_path+"/values.yaml"
+        if not os.path.exists(values_file_path):
+            raise Exception("File %s not existed" % values_file_path)
+        utils.update_values(values_file_path, args.hostname)
+        try:
+            logger.debug("Start install server")
+            self.host, self.ip = utils.helm_install_server(helm_path, args.image_tag, args.image_type, self.name, namespace)
+        except Exception as e:
+            logger.error("Helm install server failed: %s" % str(e))
+            logger.error(traceback.format_exc())
+            self.clean_up()
+            return False
+        # for debugging
+        # self.host = "192.168.1.101"
+        if not self.host:
+            logger.error("Helm install server failed")
+            self.clean_up()
+            return False
+        return True
+
+    def clean_up(self):
+        logger.debug(self.name)
+        utils.helm_del_server(self.name)
+
+    def report_wrapper(self, milvus_instance, env_value, hostname, table_info, index_info, search_params):
+        metric = Metric()
+        metric.set_run_id(timestamp)
+        metric.env = Env(env_value)
+        metric.env.OMP_NUM_THREADS = 0
+        metric.hardware = Hardware(name=hostname)
+        server_version = milvus_instance.get_server_version()
+        server_mode = milvus_instance.get_server_mode()
+        commit = milvus_instance.get_server_commit()
+        metric.server = Server(version=server_version, mode=server_mode, build_commit=commit)
+        metric.table = table_info
+        metric.index = index_info
+        metric.search = search_params
+        return metric
+
+    def run(self, run_type, table):
+        logger.debug(run_type)
+        logger.debug(table)
+        table_name = table["table_name"]
+        milvus_instance = MilvusClient(table_name=table_name, ip=self.ip)
+        self.env_value = milvus_instance.get_server_config()
+        if run_type == "insert_performance":
+            (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
+            ni_per = table["ni_per"]
+            build_index = table["build_index"]
+            if milvus_instance.exists_table():
+                milvus_instance.delete()
+                time.sleep(10)
+            index_info = {}
+            search_params = {}
+            milvus_instance.create_table(table_name, dimension, index_file_size, metric_type)
+            if build_index is True:
+                index_type = table["index_type"]
+                nlist = table["nlist"]
+                index_info = {
+                    "index_type": index_type,
+                    "index_nlist": nlist
+                }
+                milvus_instance.create_index(index_type, nlist)
+            res = self.do_insert(milvus_instance, table_name, data_type, dimension, table_size, ni_per)
+            logger.info(res)
+            table_info = {
+                "dimension": dimension,
+                "metric_type": metric_type,
+                "dataset_name": table_name
+            }
+            metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_params)
+            metric.metrics = {
+                "type": "insert_performance",
+                "value": {
+                    "total_time": res["total_time"],
+                    "qps": res["qps"],
+                    "ni_time": res["ni_time"]
+                } 
+            }
+            report(metric)
+            logger.debug("Wait for file merge")
+            time.sleep(120)
+
+        elif run_type == "build_performance":
+            (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
+            index_type = table["index_type"]
+            nlist = table["nlist"]
+            table_info = {
+                "dimension": dimension,
+                "metric_type": metric_type,
+                "dataset_name": table_name
+            }
+            index_info = {
+                "index_type": index_type,
+                "index_nlist": nlist
+            }
+            if not milvus_instance.exists_table():
+                logger.error("Table name: %s not existed" % table_name)
+                return
+            search_params = {}
+            start_time = time.time()
+            # drop index
+            logger.debug("Drop index")
+            milvus_instance.drop_index()
+            start_mem_usage = milvus_instance.get_mem_info()["memory_used"]
+            milvus_instance.create_index(index_type, nlist)
+            logger.debug(milvus_instance.describe_index())
+            end_time = time.time()
+            end_mem_usage = milvus_instance.get_mem_info()["memory_used"]
+            metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_params)
+            metric.metrics = {
+                "type": "build_performance",
+                "value": {
+                    "build_time": round(end_time - start_time, 1),
+                    "start_mem_usage": start_mem_usage,
+                    "end_mem_usage": end_mem_usage,
+                    "diff_mem": end_mem_usage - start_mem_usage
+                } 
+            }
+            report(metric)
+
+        elif run_type == "search_performance":
+            (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
+            run_count = table["run_count"]
+            search_params = table["search_params"]
+            table_info = {
+                "dimension": dimension,
+                "metric_type": metric_type,
+                "dataset_name": table_name
+            }
+            if not milvus_instance.exists_table():
+                logger.error("Table name: %s not existed" % table_name)
+                return
+            logger.info(milvus_instance.count())
+            result = milvus_instance.describe_index()
+            index_info = {
+                "index_type": result["index_type"],
+                "index_nlist": result["nlist"]
+            }
+            logger.info(index_info)
+            nprobes = search_params["nprobes"]
+            top_ks = search_params["top_ks"]
+            nqs = search_params["nqs"]
+            milvus_instance.preload_table()
+            logger.info("Start warm up query")
+            res = self.do_query(milvus_instance, table_name, [1], [1], 1, 2)
+            logger.info("End warm up query")
+            for nprobe in nprobes:
+                logger.info("Search nprobe: %s" % nprobe)
+                res = self.do_query(milvus_instance, table_name, top_ks, nqs, nprobe, run_count)
+                headers = ["Nq/Top-k"]
+                headers.extend([str(top_k) for top_k in top_ks])
+                utils.print_table(headers, nqs, res)
+                for index_nq, nq in enumerate(nqs):
+                    for index_top_k, top_k in enumerate(top_ks):
+                        search_param = {
+                            "nprobe": nprobe,
+                            "nq": nq,
+                            "topk": top_k
+                        }
+                        search_time = res[index_nq][index_top_k]
+                        metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_param)
+                        metric.metrics = {
+                            "type": "search_performance",
+                            "value": {
+                                "search_time": search_time
+                            } 
+                        }
+                        report(metric)
+
+        # for sift/deep datasets
+        elif run_type == "accuracy":
+            (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
+            search_params = table["search_params"]
+            table_info = {
+                "dimension": dimension,
+                "metric_type": metric_type,
+                "dataset_name": table_name
+            }
+            if not milvus_instance.exists_table():
+                logger.error("Table name: %s not existed" % table_name)
+                return
+            logger.info(milvus_instance.count())
+            result = milvus_instance.describe_index()
+            index_info = {
+                "index_type": result["index_type"],
+                "index_nlist": result["nlist"]
+            }
+            logger.info(index_info)
+            nprobes = search_params["nprobes"]
+            top_ks = search_params["top_ks"]
+            nqs = search_params["nqs"]
+            milvus_instance.preload_table()
+            true_ids_all = self.get_groundtruth_ids(table_size)
+            for nprobe in nprobes:
+                logger.info("Search nprobe: %s" % nprobe)
+                for top_k in top_ks:
+                    for nq in nqs:
+                        total = 0
+                        search_param = {
+                            "nprobe": nprobe,
+                            "nq": nq,
+                            "topk": top_k
+                        }
+                        result_ids, result_distances = self.do_query_ids(milvus_instance, table_name, top_k, nq, nprobe)
+                        acc_value = self.get_recall_value(true_ids_all[:nq, :top_k].tolist(), result_ids)
+                        logger.info("Query accuracy: %s" % acc_value)
+                        metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_param)
+                        metric.metrics = {
+                            "type": "accuracy",
+                            "value": {
+                                "acc": acc_value
+                            } 
+                        }
+                        report(metric)
+
+        elif run_type == "ann_accuracy":
+            hdf5_source_file = table["source_file"]
+            table_name = table["table_name"]
+            index_file_sizes = table["index_file_sizes"]
+            index_types = table["index_types"]
+            nlists = table["nlists"]
+            search_params = table["search_params"]
+            nprobes = search_params["nprobes"]
+            top_ks = search_params["top_ks"]
+            nqs = search_params["nqs"]
+            data_type, dimension, metric_type = parser.parse_ann_table_name(table_name)
+            table_info = {
+                "dimension": dimension,
+                "metric_type": metric_type,
+                "dataset_name": table_name
+            }
+            dataset = utils.get_dataset(hdf5_source_file)
+            if milvus_instance.exists_table(table_name):
+                logger.info("Re-create table: %s" % table_name)
+                milvus_instance.delete(table_name)
+                time.sleep(DELETE_INTERVAL_TIME)
+            true_ids = np.array(dataset["neighbors"])
+            for index_file_size in index_file_sizes:
+                milvus_instance.create_table(table_name, dimension, index_file_size, metric_type)
+                logger.info(milvus_instance.describe())
+                insert_vectors = self.normalize(metric_type, np.array(dataset["train"]))
+                # Insert batch once
+                # milvus_instance.insert(insert_vectors)
+                loops = len(insert_vectors) // INSERT_INTERVAL + 1
+                for i in range(loops):
+                    start = i*INSERT_INTERVAL
+                    end = min((i+1)*INSERT_INTERVAL, len(insert_vectors))
+                    tmp_vectors = insert_vectors[start:end]
+                    if start < end:
+                        if not isinstance(tmp_vectors, list):
+                            milvus_instance.insert(tmp_vectors.tolist(), ids=[i for i in range(start, end)])
+                        else:
+                            milvus_instance.insert(tmp_vectors, ids=[i for i in range(start, end)])
+                time.sleep(20)
+                logger.info("Table: %s, row count: %s" % (table_name, milvus_instance.count()))
+                if milvus_instance.count() != len(insert_vectors):
+                    logger.error("Table row count is not equal to insert vectors")
+                    return
+                for index_type in index_types:
+                    for nlist in nlists:
+                        milvus_instance.create_index(index_type, nlist)
+                        # logger.info(milvus_instance.describe_index())
+                        logger.info("Start preload table: %s, index_type: %s, nlist: %s" % (table_name, index_type, nlist))
+                        milvus_instance.preload_table()
+                        index_info = {
+                            "index_type": index_type,
+                            "index_nlist": nlist
+                        }
+                        for nprobe in nprobes:
+                            for nq in nqs:
+                                query_vectors = self.normalize(metric_type, np.array(dataset["test"][:nq]))
+                                for top_k in top_ks:
+                                    search_params = {
+                                        "nq": len(query_vectors),
+                                        "nprobe": nprobe,
+                                        "topk": top_k
+                                    }
+                                    if not isinstance(query_vectors, list):
+                                        result = milvus_instance.query(query_vectors.tolist(), top_k, nprobe)
+                                    else:
+                                        result = milvus_instance.query(query_vectors, top_k, nprobe)
+                                    result_ids = result.id_array
+                                    acc_value = self.get_recall_value(true_ids[:nq, :top_k].tolist(), result_ids)
+                                    logger.info("Query ann_accuracy: %s" % acc_value)
+                                    metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_params)
+                                    metric.metrics = {
+                                        "type": "ann_accuracy",
+                                        "value": {
+                                            "acc": acc_value
+                                        } 
+                                    }
+                                    report(metric)
+                milvus_instance.delete()
+
+        elif run_type == "search_stability":
+            (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
+            search_params = table["search_params"]
+            during_time = table["during_time"]
+            table_info = {
+                "dimension": dimension,
+                "metric_type": metric_type,
+                "dataset_name": table_name
+            }
+            if not milvus_instance.exists_table():
+                logger.error("Table name: %s not existed" % table_name)
+                return
+            logger.info(milvus_instance.count())
+            result = milvus_instance.describe_index()
+            index_info = {
+                "index_type": result["index_type"],
+                "index_nlist": result["nlist"]
+            }
+            search_param = {}
+            logger.info(index_info)
+            g_nprobe = int(search_params["nprobes"].split("-")[1])
+            g_top_k = int(search_params["top_ks"].split("-")[1])
+            g_nq = int(search_params["nqs"].split("-")[1])
+            l_nprobe = int(search_params["nprobes"].split("-")[0])
+            l_top_k = int(search_params["top_ks"].split("-")[0])
+            l_nq = int(search_params["nqs"].split("-")[0])
+            milvus_instance.preload_table()
+            start_mem_usage = milvus_instance.get_mem_info()["memory_used"]
+            logger.debug(start_mem_usage)
+            logger.info("Start warm up query")
+            res = self.do_query(milvus_instance, table_name, [1], [1], 1, 2)
+            logger.info("End warm up query")
+            start_time = time.time()
+            while time.time() < start_time + during_time * 60:
+                top_k = random.randint(l_top_k, g_top_k)
+                nq = random.randint(l_nq, g_nq)
+                nprobe = random.randint(l_nprobe, g_nprobe)
+                query_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)]
+                logger.debug("Query nprobe:%d, nq:%d, top-k:%d" % (nprobe, nq, top_k))
+                result = milvus_instance.query(query_vectors, top_k, nprobe)
+            end_mem_usage = milvus_instance.get_mem_info()["memory_used"]
+            metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_param)
+            metric.metrics = {
+                "type": "search_stability",
+                "value": {
+                    "during_time": during_time,
+                    "start_mem_usage": start_mem_usage,
+                    "end_mem_usage": end_mem_usage,
+                    "diff_mem": end_mem_usage - start_mem_usage
+                } 
+            }
+            report(metric)
+
+        elif run_type == "stability":
+            (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
+            search_params = table["search_params"]
+            insert_xb = table["insert_xb"]
+            insert_interval = table["insert_interval"]
+            during_time = table["during_time"]
+            table_info = {
+                "dimension": dimension,
+                "metric_type": metric_type,
+                "dataset_name": table_name
+            }
+            if not milvus_instance.exists_table():
+                logger.error("Table name: %s not existed" % table_name)
+                return
+            logger.info(milvus_instance.count())
+            result = milvus_instance.describe_index()
+            index_info = {
+                "index_type": result["index_type"],
+                "index_nlist": result["nlist"]
+            }
+            search_param = {}
+            logger.info(index_info)
+            g_nprobe = int(search_params["nprobes"].split("-")[1])
+            g_top_k = int(search_params["top_ks"].split("-")[1])
+            g_nq = int(search_params["nqs"].split("-")[1])
+            l_nprobe = int(search_params["nprobes"].split("-")[0])
+            l_top_k = int(search_params["top_ks"].split("-")[0])
+            l_nq = int(search_params["nqs"].split("-")[0])
+            milvus_instance.preload_table()
+            logger.info("Start warm up query")
+            res = self.do_query(milvus_instance, table_name, [1], [1], 1, 2)
+            logger.info("End warm up query")
+            start_mem_usage = milvus_instance.get_mem_info()["memory_used"]
+            start_row_count = milvus_instance.count()
+            start_time = time.time()
+            i = 0
+            while time.time() < start_time + during_time * 60:
+                i = i + 1
+                for j in range(insert_interval):
+                    top_k = random.randint(l_top_k, g_top_k)
+                    nq = random.randint(l_nq, g_nq)
+                    nprobe = random.randint(l_nprobe, g_nprobe)
+                    query_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)]
+                    logger.debug("Query nprobe:%d, nq:%d, top-k:%d" % (nprobe, nq, top_k))
+                    result = milvus_instance.query(query_vectors, top_k, nprobe)
+                insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)]
+                status, res = milvus_instance.insert(insert_vectors, ids=[x for x in range(len(insert_vectors))])
+                logger.debug("%d, row_count: %d" % (i, milvus_instance.count()))
+            end_mem_usage = milvus_instance.get_mem_info()["memory_used"]
+            end_row_count = milvus_instance.count()
+            metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_param)
+            metric.metrics = {
+                "type": "stability",
+                "value": {
+                    "during_time": during_time,
+                    "start_mem_usage": start_mem_usage,
+                    "end_mem_usage": end_mem_usage,
+                    "diff_mem": end_mem_usage - start_mem_usage,
+                    "row_count_increments": end_row_count - start_row_count
+                } 
+            }
+            report(metric)

tests/milvus_benchmark/local_runner.py

@@ -48,6 +48,8 @@ class LocalRunner(Runner):
                         milvus = MilvusClient(table_name, ip=self.ip, port=self.port)
                         logger.info(milvus.describe())
                         logger.info(milvus.describe_index())
+                        logger.info(milvus.count())
+                        logger.info(milvus.show_tables())
                         # parse index info
                         index_types = param["index.index_types"]
                         nlists = param["index.nlists"]
@@ -64,7 +66,7 @@ class LocalRunner(Runner):
                                 logger.info("End preloading table")
                                 # Run query test
                                 logger.info("Start warm up query")
-                                res = self.do_query(milvus, table_name, [1], [1], 1, 1)
+                                res = self.do_query(milvus, table_name, [1], [1], 1, 2)
                                 logger.info("End warm up query")
                                 for nprobe in nprobes:
                                     logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe))
@@ -204,12 +206,14 @@ class LocalRunner(Runner):
                         #     p.join()
                         i = i + 1
                         milvus_instance = MilvusClient(table_name, ip=self.ip, port=self.port)
-                        top_ks = random.sample([x for x in range(1, 100)], 4)
-                        nqs = random.sample([x for x in range(1, 200)], 3)
+                        top_ks = random.sample([x for x in range(1, 100)], 1)
+                        nqs = random.sample([x for x in range(1, 200)], 2)
                         nprobe = random.choice([x for x in range(1, 100)])
                         res = self.do_query(milvus_instance, table_name, top_ks, nqs, nprobe, run_count)
                         # milvus_instance = MilvusClient(table_name)
                         status, res = milvus_instance.insert(insert_vectors, ids=[x for x in range(len(insert_vectors))])
                         if not status.OK():
                             logger.error(status.message)
+                        logger.debug(milvus.count())
+                        res = self.do_query(milvus_instance, table_name, top_ks, nqs, nprobe, run_count)
                         # status = milvus_instance.create_index(index_type, 16384)

tests/milvus_benchmark/main.py

@@ -4,26 +4,31 @@ import time
 import pdb
 import argparse
 import logging
-import utils
-from yaml import load, dump
+import traceback
 from logging import handlers
+
+from yaml import full_load, dump
 from parser import operations_parser
 from local_runner import LocalRunner
 from docker_runner import DockerRunner
+from k8s_runner import K8sRunner
 
 DEFAULT_IMAGE = "milvusdb/milvus:latest"
-LOG_FOLDER = "benchmark_logs"
-logger = logging.getLogger("milvus_benchmark")
+LOG_FOLDER = "logs"
 
-formatter = logging.Formatter('[%(asctime)s] [%(levelname)-4s] [%(pathname)s:%(lineno)d] %(message)s')
-if not os.path.exists(LOG_FOLDER):
-    os.system('mkdir -p %s' % LOG_FOLDER)
-fileTimeHandler = handlers.TimedRotatingFileHandler(os.path.join(LOG_FOLDER, 'milvus_benchmark'), "D", 1, 10)
-fileTimeHandler.suffix = "%Y%m%d.log"
-fileTimeHandler.setFormatter(formatter)
-logging.basicConfig(level=logging.DEBUG)
-fileTimeHandler.setFormatter(formatter)
-logger.addHandler(fileTimeHandler)
+# formatter = logging.Formatter('[%(asctime)s] [%(levelname)-4s] [%(pathname)s:%(lineno)d] %(message)s')
+# if not os.path.exists(LOG_FOLDER):
+#     os.system('mkdir -p %s' % LOG_FOLDER)
+# fileTimeHandler = handlers.TimedRotatingFileHandler(os.path.join(LOG_FOLDER, 'milvus_benchmark'), "D", 1, 10)
+# fileTimeHandler.suffix = "%Y%m%d.log"
+# fileTimeHandler.setFormatter(formatter)
+# logging.basicConfig(level=logging.DEBUG)
+# fileTimeHandler.setFormatter(formatter)
+# logger.addHandler(fileTimeHandler)
+logging.basicConfig(format='%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
+    datefmt='%Y-%m-%d:%H:%M:%S',
+    level=logging.DEBUG)
+logger = logging.getLogger("milvus_benchmark")
 
 
 def positive_int(s):
@@ -51,30 +56,39 @@ def main():
     parser = argparse.ArgumentParser(
         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument(
-        '--image',
-        help='use the given image')
+        "--hostname",
+        default="eros",
+        help="server host name")
+    parser.add_argument(
+        "--image-tag",
+        default="",
+        help="image tag")
+    parser.add_argument(
+        "--image-type",
+        default="",
+        help="image type")
+    # parser.add_argument(
+    #     "--run-count",
+    #     default=1,
+    #     type=positive_int,
+    #     help="run times for each test")
+    # # performance / stability / accuracy test
+    # parser.add_argument(
+    #     "--run-type",
+    #     default="search_performance",
+    #     help="run type, default performance")
+    parser.add_argument(
+        '--suite',
+        metavar='FILE',
+        help='load test suite from FILE',
+        default='suites/suite.yaml')
     parser.add_argument(
         '--local',
         action='store_true',
         help='use local milvus server')
     parser.add_argument(
-        "--run-count",
-        default=1,
-        type=positive_int,
-        help="run each db operation times")
-    # performance / stability / accuracy test
-    parser.add_argument(
-        "--run-type",
-        default="performance",
-        help="run type, default performance")
-    parser.add_argument(
-        '--suites',
-        metavar='FILE',
-        help='load test suites from FILE',
-        default='suites.yaml')
-    parser.add_argument(
-        '--ip',
-        help='server ip param for local mode',
+        '--host',
+        help='server host ip param for local mode',
         default='127.0.0.1')
     parser.add_argument(
         '--port',
@@ -83,43 +97,60 @@ def main():
 
     args = parser.parse_args()
 
-    operations = None
     # Get all benchmark test suites
-    if args.suites:
-        with open(args.suites) as f:
-            suites_dict = load(f)
+    if args.suite:
+        with open(args.suite) as f:
+            suite_dict = full_load(f)
             f.close()
         # With definition order
-        operations = operations_parser(suites_dict, run_type=args.run_type)
+        run_type, run_params = operations_parser(suite_dict)
 
     # init_env()
-    run_params = {"run_count": args.run_count}
+    # run_params = {"run_count": args.run_count}
 
-    if args.image:
+    if args.image_tag:
+        namespace = "milvus"
+        logger.debug(args)
         # for docker mode
         if args.local:
-            logger.error("Local mode and docker mode are incompatible arguments")
+            logger.error("Local mode and docker mode are incompatible")
             sys.exit(-1)
         # Docker pull image
-        if not utils.pull_image(args.image):
-            raise Exception('Image %s pull failed' % image)
-
+        # if not utils.pull_image(args.image):
+        #     raise Exception('Image %s pull failed' % image)
         # TODO: Check milvus server port is available
-        logger.info("Init: remove all containers created with image: %s" % args.image)
-        utils.remove_all_containers(args.image)
-        runner = DockerRunner(args.image)
-        for operation_type in operations:
-            logger.info("Start run test, test type: %s" % operation_type)
-            run_params["params"] = operations[operation_type]
-            runner.run({operation_type: run_params}, run_type=args.run_type)
-            logger.info("Run params: %s" % str(run_params))
+        # logger.info("Init: remove all containers created with image: %s" % args.image)
+        # utils.remove_all_containers(args.image)
+        # runner = DockerRunner(args)
+        tables = run_params["tables"]
+        for table in tables:
+            # run tests
+            server_config = table["server"]
+            logger.debug(server_config)
+            runner = K8sRunner()
+            if runner.init_env(server_config, args):
+                logger.debug("Start run tests")
+                try:
+                    runner.run(run_type, table)
+                except Exception as e:
+                    logger.error(str(e))
+                    logger.error(traceback.format_exc())
+                finally:
+                    runner.clean_up()
+            else:
+                logger.error("Runner init failed")
+        # for operation_type in operations:
+        #     logger.info("Start run test, test type: %s" % operation_type)
+        #     run_params["params"] = operations[operation_type]
+        #     runner.run({operation_type: run_params}, run_type=args.run_type)
+        #     logger.info("Run params: %s" % str(run_params))
 
     if args.local:
         # for local mode
-        ip = args.ip
+        host = args.host
         port = args.port
 
-        runner = LocalRunner(ip, port)
+        runner = LocalRunner(host, port)
         for operation_type in operations:
             logger.info("Start run local mode test, test type: %s" % operation_type)
             run_params["params"] = operations[operation_type]

tests/milvus_benchmark/parser.py

@@ -4,9 +4,12 @@ import logging
 logger = logging.getLogger("milvus_benchmark.parser")
 
 
-def operations_parser(operations, run_type="performance"):
-    definitions = operations[run_type]
-    return definitions
+def operations_parser(operations):
+    if not operations:
+        raise Exception("No operations in suite defined")
+    for run_type, run_params in operations.items():
+        logger.debug(run_type)
+        return (run_type, run_params)
 
 
 def table_parser(table_name):
@@ -26,6 +29,23 @@ def table_parser(table_name):
     return (data_type, table_size, index_file_size, dimension, metric_type)
 
 
+def parse_ann_table_name(table_name):
+    data_type = table_name.split("_")[0]
+    dimension = int(table_name.split("_")[1])
+    metric = table_name.split("_")[-1]
+    # metric = table_name.attrs['distance']
+    # dimension = len(table_name["train"][0])
+    if metric == "euclidean":
+        metric_type = "l2"
+    elif metric  == "angular":
+        metric_type = "ip"
+    elif metric  == "jaccard":
+        metric_type = "jaccard"
+    elif metric == "hamming":
+        metric_type = "hamming"
+    return ("ann_"+data_type, dimension, metric_type)
+
+
 def search_params_parser(param):
     # parse top-k, set default value if top-k not in param
     if "top_ks" not in param:

tests/milvus_benchmark/requirements.txt

@@ -2,8 +2,9 @@ numpy==1.16.3
 pymilvus-test>=0.2.0
 scikit-learn==0.19.1
 h5py==2.7.1
-influxdb==5.2.2
+# influxdb==5.2.2
 pyyaml>=5.1
 tableprint==0.8.0
 ansicolors==1.1.8
 scipy==1.3.1
+kubernetes==10.0.1

tests/milvus_benchmark/runner.py

@@ -5,24 +5,26 @@ import time
 import random
 from multiprocessing import Process
 import numpy as np
+import sklearn.preprocessing
 from client import MilvusClient
 import utils
 import parser
 
 logger = logging.getLogger("milvus_benchmark.runner")
 
-SERVER_HOST_DEFAULT = "127.0.0.1"
-SERVER_PORT_DEFAULT = 19530
 VECTORS_PER_FILE = 1000000
 SIFT_VECTORS_PER_FILE = 100000
+JACCARD_VECTORS_PER_FILE = 2000000
+
 MAX_NQ = 10001
 FILE_PREFIX = "binary_"
 
 # FOLDER_NAME = 'ann_1000m/source_data'
-SRC_BINARY_DATA_DIR = '/poc/yuncong/yunfeng/random_data'
-SRC_BINARY_DATA_DIR_high = '/test/milvus/raw_data/random'
-SIFT_SRC_DATA_DIR = '/poc/yuncong/ann_1000m/'
-SIFT_SRC_BINARY_DATA_DIR = SIFT_SRC_DATA_DIR + 'source_data'
+SRC_BINARY_DATA_DIR = '/test/milvus/raw_data/random/'
+SIFT_SRC_DATA_DIR = '/test/milvus/raw_data/sift1b/'
+DEEP_SRC_DATA_DIR = '/test/milvus/raw_data/deep1b/'
+JACCARD_SRC_DATA_DIR = '/test/milvus/raw_data/jaccard/'
+HAMMING_SRC_DATA_DIR = '/test/milvus/raw_data/jaccard/'
 SIFT_SRC_GROUNDTRUTH_DATA_DIR = SIFT_SRC_DATA_DIR + 'gnd'
 
 WARM_TOP_K = 1
@@ -44,16 +46,19 @@ GROUNDTRUTH_MAP = {
 }
 
 
-def gen_file_name(idx, table_dimension, data_type):
+def gen_file_name(idx, dimension, data_type):
     s = "%05d" % idx
-    fname = FILE_PREFIX + str(table_dimension) + "d_" + s + ".npy"
+    fname = FILE_PREFIX + str(dimension) + "d_" + s + ".npy"
     if data_type == "random":
-        if table_dimension == 512:
-            fname = SRC_BINARY_DATA_DIR+'/'+fname
-        elif table_dimension >= 4096:
-            fname = SRC_BINARY_DATA_DIR_high+'/'+fname
+        fname = SRC_BINARY_DATA_DIR+fname
     elif data_type == "sift":
-        fname = SIFT_SRC_BINARY_DATA_DIR+'/'+fname
+        fname = SIFT_SRC_DATA_DIR+fname
+    elif data_type == "deep":
+        fname = DEEP_SRC_DATA_DIR+fname
+    elif data_type == "jaccard":
+        fname = JACCARD_SRC_DATA_DIR+fname
+    elif data_type == "hamming":
+        fname = HAMMING_SRC_DATA_DIR+fname
     return fname
 
 
@@ -62,9 +67,15 @@ def get_vectors_from_binary(nq, dimension, data_type):
     if nq > MAX_NQ:
         raise Exception("Over size nq")
     if data_type == "random":
-        file_name = gen_file_name(0, dimension, data_type)
+        file_name = SRC_BINARY_DATA_DIR+'query_%d.npy' % dimension
     elif data_type == "sift":
-        file_name = SIFT_SRC_DATA_DIR+'/'+'query.npy'
+        file_name = SIFT_SRC_DATA_DIR+'query.npy'
+    elif data_type == "deep":
+        file_name = DEEP_SRC_DATA_DIR+'query.npy'
+    elif data_type == "jaccard":
+        file_name = JACCARD_SRC_DATA_DIR+'query.npy'
+    elif data_type == "hamming":
+        file_name = HAMMING_SRC_DATA_DIR+'query.npy'
     data = np.load(file_name)
     vectors = data[0:nq].tolist()
     return vectors
@@ -74,6 +85,21 @@ class Runner(object):
     def __init__(self):
         pass
 
+    def normalize(self, metric_type, X):
+        if metric_type == "ip":
+            logger.info("Set normalize for metric_type: %s" % metric_type)
+            X = sklearn.preprocessing.normalize(X, axis=1, norm='l2')
+            X = X.astype(np.float32)
+        elif metric_type == "l2":
+            X = X.astype(np.float32)
+        elif metric_type == "jaccard" or metric_type == "hamming":
+            tmp = []
+            for index, item in enumerate(X):
+                new_vector = bytes(np.packbits(item, axis=-1).tolist())
+                tmp.append(new_vector)
+            X = tmp
+        return X
+
     def do_insert(self, milvus, table_name, data_type, dimension, size, ni):
         '''
         @params:
@@ -101,14 +127,18 @@ class Runner(object):
                 vectors_per_file = 10000
         elif data_type == "sift":
             vectors_per_file = SIFT_VECTORS_PER_FILE
+        elif data_type == "jaccard" or data_type == "hamming":
+            vectors_per_file = JACCARD_VECTORS_PER_FILE
+        else:
+            raise Exception("data_type: %s not supported" % data_type)
         if size % vectors_per_file or ni > vectors_per_file:
             raise Exception("Not invalid table size or ni")
         file_num = size // vectors_per_file
         for i in range(file_num):
             file_name = gen_file_name(i, dimension, data_type)
-            logger.info("Load npy file: %s start" % file_name)
+            # logger.info("Load npy file: %s start" % file_name)
             data = np.load(file_name)
-            logger.info("Load npy file: %s end" % file_name)
+            # logger.info("Load npy file: %s end" % file_name)
             loops = vectors_per_file // ni
             for j in range(loops):
                 vectors = data[j*ni:(j+1)*ni].tolist()
@@ -130,28 +160,26 @@ class Runner(object):
         bi_res["ni_time"] = ni_time
         return bi_res
 
-    def do_query(self, milvus, table_name, top_ks, nqs, nprobe, run_count):
+    def do_query(self, milvus, table_name, top_ks, nqs, nprobe, run_count=1):
+        bi_res = []
         (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
         base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type)
-
-        bi_res = []
-        for index, nq in enumerate(nqs):
+        for nq in nqs:
             tmp_res = []
             vectors = base_query_vectors[0:nq]
             for top_k in top_ks:
                 avg_query_time = 0.0
-                total_query_time = 0.0
+                min_query_time = 0.0
                 logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(vectors)))
                 for i in range(run_count):
                     logger.info("Start run query, run %d of %s" % (i+1, run_count))
                     start_time = time.time()
-                    status, query_res = milvus.query(vectors, top_k, nprobe)
-                    total_query_time = total_query_time + (time.time() - start_time)
-                    if status.code:
-                        logger.error("Query failed with message: %s" % status.message)
-                avg_query_time = round(total_query_time / run_count, 2)
-                logger.info("Avarage query time: %.2f" % avg_query_time)
-                tmp_res.append(avg_query_time)
+                    query_res = milvus.query(vectors, top_k, nprobe)
+                    interval_time = time.time() - start_time
+                    if (i == 0) or (min_query_time > interval_time):
+                        min_query_time = interval_time
+                logger.info("Min query time: %.2f" % min_query_time)
+                tmp_res.append(round(min_query_time, 2))
             bi_res.append(tmp_res)
         return bi_res
 
@@ -160,10 +188,7 @@ class Runner(object):
         base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type)
         vectors = base_query_vectors[0:nq]
         logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(vectors)))
-        status, query_res = milvus.query(vectors, top_k, nprobe)
-        if not status.OK():
-            msg = "Query failed with message: %s" % status.message
-            raise Exception(msg)
+        query_res = milvus.query(vectors, top_k, nprobe)
         result_ids = []
         result_distances = []
         for result in query_res:
@@ -181,10 +206,7 @@ class Runner(object):
         base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type)
         vectors = base_query_vectors[0:nq]
         logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(vectors)))
-        status, query_res = milvus.query(vectors, top_k, nprobe)
-        if not status.OK():
-            msg = "Query failed with message: %s" % status.message
-            raise Exception(msg)
+        query_res = milvus.query(vectors, top_k, nprobe)
         # if file existed, cover it
         if os.path.isfile(id_store_name):
             os.remove(id_store_name)
@@ -212,15 +234,17 @@ class Runner(object):
         # get the accuracy
         return self.get_recall_value(flat_id_list, index_id_list)
 
-    def get_recall_value(self, flat_id_list, index_id_list):
+    def get_recall_value(self, true_ids, result_ids):
         """
         Use the intersection length
         """
         sum_radio = 0.0
-        for index, item in enumerate(index_id_list):
-            tmp = set(item).intersection(set(flat_id_list[index]))
+        for index, item in enumerate(result_ids):
+            # tmp = set(item).intersection(set(flat_id_list[index]))
+            tmp = set(true_ids[index]).intersection(set(item))
             sum_radio = sum_radio + len(tmp) / len(item)
-        return round(sum_radio / len(index_id_list), 3)
+            # logger.debug(sum_radio)
+        return round(sum_radio / len(result_ids), 3)
 
     """
     Implementation based on:

tests/milvus_benchmark/scripts/default_config.json

@@ -0,0 +1,86 @@
+[
+    {
+        "job_name": "milvus-apollo",
+        "build_params": {
+            "SUITE": "cpu_accuracy_ann.yaml",
+            "IMAGE_TYPE": "cpu",
+            "IMAGE_VERSION": "master",
+            "SERVER_HOST": "apollo"
+        }
+    },
+    {
+        "job_name": "milvus-apollo",
+        "build_params": {
+            "SUITE": "cpu_stability_sift50m.yaml",
+            "IMAGE_TYPE": "cpu",
+            "IMAGE_VERSION": "master",
+            "SERVER_HOST": "apollo"
+        }
+    },
+
+    {
+        "job_name": "milvus-eros",
+        "build_params": {
+            "SUITE": "gpu_accuracy_ann.yaml",
+            "IMAGE_TYPE": "gpu",
+            "IMAGE_VERSION": "master",
+            "SERVER_HOST": "eros"
+        }
+    },
+    {
+        "job_name": "milvus-eros",
+        "build_params": {
+            "SUITE": "gpu_search_stability.yaml",
+            "IMAGE_TYPE": "gpu",
+            "IMAGE_VERSION": "master",
+            "SERVER_HOST": "eros"
+        }
+    },
+    {
+        "job_name": "milvus-eros",
+        "build_params": {
+            "SUITE": "gpu_build_performance.yaml",
+            "IMAGE_TYPE": "gpu",
+            "IMAGE_VERSION": "master",
+            "SERVER_HOST": "eros"
+        }
+    },
+
+    {
+        "job_name": "milvus-poseidon",
+        "build_params": {
+            "SUITE": "gpu_search_performance.yaml",
+            "IMAGE_TYPE": "gpu",
+            "IMAGE_VERSION": "master",
+            "SERVER_HOST": "poseidon"
+        }
+    },
+    {
+        "job_name": "milvus-poseidon",
+        "build_params": {
+            "SUITE": "cpu_search_performance.yaml",
+            "IMAGE_TYPE": "cpu",
+            "IMAGE_VERSION": "master",
+            "SERVER_HOST": "poseidon"
+        }
+    },
+    {
+        "job_name": "milvus-poseidon",
+        "build_params": {
+            "SUITE": "insert_performance.yaml",
+            "IMAGE_TYPE": "gpu",
+            "IMAGE_VERSION": "master",
+            "SERVER_HOST": "poseidon"
+        }
+    },
+    {
+        "job_name": "milvus-poseidon",
+        "build_params": {
+            "SUITE": "gpu_accuracy.yaml",
+            "IMAGE_TYPE": "gpu",
+            "IMAGE_VERSION": "master",
+            "SERVER_HOST": "poseidon"
+        }
+    }
+
+]

tests/milvus_benchmark/scripts/scheduler.py

@@ -0,0 +1,28 @@
+import json
+from pprint import pprint
+import jenkins
+
+JENKINS_URL = "****"
+server = jenkins.Jenkins(JENKINS_URL, username='****', password='****')
+user = server.get_whoami()
+version = server.get_version()
+print('Hello %s from Jenkins %s' % (user['fullName'], version))
+
+
+# print(job_config)
+# build_params = {
+#     "SUITE": "gpu_accuracy_ann_debug.yaml",
+#     "IMAGE_TYPE": "cpu",
+#     "IMAGE_VERSION": "tanimoto_distance",
+#     "SERVER_HOST": "eros"
+# }
+# print(server.build_job(job_name, build_params))
+
+
+with open("default_config.json") as json_file:
+	data = json.load(json_file)
+	for config in data:
+		build_params = config["build_params"]
+		job_name = config["job_name"]
+		res = server.build_job(job_name, build_params)
+		print(job_name, res)

tests/milvus_benchmark/suites.yaml

@@ -1,38 +0,0 @@
-# data sets
-datasets:
-  hf5:
-    gist-960,sift-128
-  npy:
-    50000000-512, 100000000-512
-
-operations:
-  # interface: search_vectors
-  query:
-    # dataset: table name you have already created
-    # key starts with "server." need to reconfig and restart server, including nprpbe/nlist/use_blas_threshold/..
-    [
-      # debug
-      # {"dataset": "ip_ivfsq8_1000", "top_ks": [16], "nqs": [1], "server.nprobe": 1, "server.use_blas_threshold": 800, "server.cpu_cache_capacity": 110},
-      
-      {"dataset": "ip_ivfsq8_1000", "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], "nqs": [1, 10, 100, 500, 800, 1000], "server.nprobe": 1, "server.use_blas_threshold": 800, "server.cpu_cache_capacity": 110},
-      {"dataset": "ip_ivfsq8_1000", "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], "nqs": [1, 10, 100, 500, 800, 1000], "server.nprobe": 10, "server.use_blas_threshold": 20, "server.cpu_cache_capacity": 110},
-      {"dataset": "ip_ivfsq8_5000", "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], "nqs": [1, 10, 100, 500, 800, 1000], "server.nprobe": 1, "server.use_blas_threshold": 800, "server.cpu_cache_capacity": 110},
-      {"dataset": "ip_ivfsq8_5000", "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], "nqs": [1, 10, 100, 500, 800, 1000], "server.nprobe": 10, "server.use_blas_threshold": 20, "server.cpu_cache_capacity": 110},
-      {"dataset": "ip_ivfsq8_40000", "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], "nqs": [1, 10, 100, 500, 800, 1000], "server.nprobe": 1, "server.use_blas_threshold": 800, "server.cpu_cache_capacity": 110},
-      # {"dataset": "ip_ivfsq8_40000", "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], "nqs": [1, 10, 100, 1000], "server.nprobe": 10, "server.use_blas_threshold": 20, "server.cpu_cache_capacity": 110},
-    ]
-
-  # interface: add_vectors
-  insert:
-    # index_type: flat/ivf_flat/ivf_sq8
-    [
-      # debug
-
-
-      {"table_name": "ip_ivf_flat_20m_1024", "table.index_type": "ivf_flat", "server.index_building_threshold": 1024, "table.size": 20000000, "table.ni": 100000, "table.dim": 512, "server.cpu_cache_capacity": 110},
-      {"table_name": "ip_ivf_sq8_50m_1024", "table.index_type": "ivf_sq8", "server.index_building_threshold": 1024, "table.size": 50000000, "table.ni": 100000, "table.dim": 512, "server.cpu_cache_capacity": 110},
-    ]
-
-  # TODO: interface: build_index
-  build: []
-

tests/milvus_benchmark/suites/cpu_accuracy.yaml

@@ -0,0 +1,119 @@
+accuracy:
+  tables:
+    # sift1b
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8
+        cache_config.cpu_cache_capacity: 150
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_1b_2048_128_l2
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [64]
+        nqs: [1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq
+        cache_config.cpu_cache_capacity: 150
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_1b_2048_128_l2
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [64]
+        nqs: [1000]
+
+    # sift50m
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf
+        cache_config.cpu_cache_capacity: 60
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [64]
+        nqs: [1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8
+        cache_config.cpu_cache_capacity: 30
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [64]
+        nqs: [1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq
+        cache_config.cpu_cache_capacity: 30
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [64]
+        nqs: [1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg
+        cache_config.cpu_cache_capacity: 100
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      search_params:
+        nprobes: [1]
+        top_ks: [64]
+        nqs: [1000]

tests/milvus_benchmark/suites/cpu_accuracy_ann.yaml

@@ -0,0 +1,68 @@
+ann_accuracy:
+  tables:
+    -
+      server:
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5
+      table_name: sift_128_euclidean
+      index_file_sizes: [1024]
+      index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_pq', 'nsg']
+      nlists: [16384]
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [10]
+        nqs: [10000]
+    -
+      server:
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/gist-960-euclidean.hdf5
+      table_name: gist_960_euclidean
+      index_file_sizes: [1024]
+      index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_pq', 'nsg']
+      nlists: [16384]
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [10]
+        nqs: [10000]
+    -
+      server:
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5
+      table_name: glove_200_angular
+      index_file_sizes: [1024]
+      index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_pq', 'nsg']
+      nlists: [16384]
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [10]
+        nqs: [10000]

tests/milvus_benchmark/suites/cpu_build_performance.yaml

@@ -0,0 +1,36 @@
+build_performance:
+  tables:
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096
+    #     cache_config.cpu_cache_capacity: 32
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 1
+    #     gpu_resource_config.enable: false
+    #     gpu_resource_config.cache_capacity: 6
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   table_name: sift_50m_1024_128_l2
+    #   index_type: ivf_sq8
+    #   nlist: 4096
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_8192
+        cache_config.cpu_cache_capacity: 32
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 6
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      index_type: ivf_sq8
+      nlist: 8192

tests/milvus_benchmark/suites/cpu_search_performance.yaml

@@ -0,0 +1,169 @@
+search_performance:
+  tables:
+    # sift_50m
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf
+        cache_config.cpu_cache_capacity: 32
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: sift_50m_1024_128_l2
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: sift_50m_1024_128_l2
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq
+        cache_config.cpu_cache_capacity: 32
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: sift_50m_1024_128_l2
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg
+        cache_config.cpu_cache_capacity: 50
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: sift_50m_1024_128_l2
+      run_count: 2
+      search_params:
+        nprobes: [8]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+
+    # random_50m
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_ivf
+        cache_config.cpu_cache_capacity: 110
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: random_50m_1024_512_ip
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8
+        cache_config.cpu_cache_capacity: 30
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: random_50m_1024_512_ip
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_nsg
+        cache_config.cpu_cache_capacity: 200
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: random_50m_1024_512_ip
+      run_count: 2
+      search_params:
+        nprobes: [8]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+
+    # sift_1b
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8
+        cache_config.cpu_cache_capacity: 150
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: sift_1b_2048_128_l2
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq
+        cache_config.cpu_cache_capacity: 150
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: sift_1b_2048_128_l2
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]

tests/milvus_benchmark/suites/cpu_search_stability.yaml

@@ -0,0 +1,20 @@
+search_stability:
+  tables:
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq
+        cache_config.cpu_cache_capacity: 50
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: sift_50m_1024_128_l2
+      during_time: 240
+      search_params:
+        nprobes: 1-200
+        top_ks: 1-200
+        nqs: 1-200

tests/milvus_benchmark/suites/cpu_stability_sift50m.yaml

@@ -0,0 +1,27 @@
+stability:
+  tables:
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_8192_stability
+        cache_config.cpu_cache_capacity: 64
+        cache_config.cache_insert_data: true
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      during_time: 480
+      search_params:
+        nprobes: 1-200
+        top_ks: 1-200
+        nqs: 1-200
+      # length of insert vectors
+      insert_xb: 100000
+      # insert after search 4 times
+      insert_interval: 4

tests/milvus_benchmark/suites/gpu_accuracy.yaml

@@ -0,0 +1,157 @@
+accuracy:
+  tables:
+    # sift1b
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8
+        cache_config.cpu_cache_capacity: 150
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_1b_2048_128_l2
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [64]
+        nqs: [1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8h
+        cache_config.cpu_cache_capacity: 150
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_1b_2048_128_l2
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [64]
+        nqs: [1000]
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq
+    #     cache_config.cpu_cache_capacity: 150
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 1
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   table_name: sift_1b_2048_128_l2
+    #   search_params:
+    #     nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+    #     top_ks: [64]
+    #     nqs: [1000]
+
+    # sift50m
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf
+        cache_config.cpu_cache_capacity: 60
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [64]
+        nqs: [1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8
+        cache_config.cpu_cache_capacity: 30
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [64]
+        nqs: [1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h
+        cache_config.cpu_cache_capacity: 30
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [64]
+        nqs: [1000]
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq
+    #     cache_config.cpu_cache_capacity: 30
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 1
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   table_name: sift_50m_1024_128_l2
+    #   search_params:
+    #     nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+    #     top_ks: [64]
+    #     nqs: [1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg
+        cache_config.cpu_cache_capacity: 100
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      search_params:
+        nprobes: [1]
+        top_ks: [64]
+        nqs: [1000]

tests/milvus_benchmark/suites/gpu_accuracy_ann.yaml

@@ -0,0 +1,68 @@
+ann_accuracy:
+  tables:
+    -
+      server:
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5
+      table_name: sift_128_euclidean
+      index_file_sizes: [1024]
+      index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h', 'ivf_pq', 'nsg']
+      nlists: [16384]
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [10]
+        nqs: [10000]
+    -
+      server:
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/gist-960-euclidean.hdf5
+      table_name: gist_960_euclidean
+      index_file_sizes: [1024]
+      index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h', 'ivf_pq', 'nsg']
+      nlists: [16384]
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [10]
+        nqs: [10000]
+    -
+      server:
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5
+      table_name: glove_200_angular
+      index_file_sizes: [1024]
+      index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h', 'nsg']
+      nlists: [16384]
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [10]
+        nqs: [10000]

tests/milvus_benchmark/suites/gpu_accuracy_ann_debug.yaml

@@ -0,0 +1,47 @@
+ann_accuracy:
+  tables:
+    -
+      server:
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/kosarak-27983-jaccard.hdf5
+      table_name: kosarak_27984_jaccard
+      index_file_sizes: [1024]
+      index_types: ['flat', 'ivf_flat']
+      nlists: [2048]
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [10]
+        nqs: [10000]
+
+    -
+      server:
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/sift-256-hamming.hdf5
+      table_name: sift_256_hamming
+      index_file_sizes: [1024]
+      index_types: ['flat', 'ivf_flat']
+      nlists: [2048]
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [100]
+        nqs: [1000]

tests/milvus_benchmark/suites/gpu_accuracy_sift1b.yaml

@@ -0,0 +1,59 @@
+accuracy:
+  tables:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8
+        cache_config.cpu_cache_capacity: 150
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_1b_2048_128_l2
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [64]
+        nqs: [1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8h
+        cache_config.cpu_cache_capacity: 150
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_1b_2048_128_l2
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [64]
+        nqs: [1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq
+        cache_config.cpu_cache_capacity: 150
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_1b_2048_128_l2
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [64]
+        nqs: [1000]

tests/milvus_benchmark/suites/gpu_accuracy_sift1m.yaml

@@ -0,0 +1,40 @@
+accuracy:
+  tables:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_ivf
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_1m_1024_128_l2
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [64]
+        nqs: [1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_sq8
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_1m_1024_128_l2
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+        top_ks: [64]
+        nqs: [1000]

tests/milvus_benchmark/suites/gpu_accuracy_sift50m.yaml

@@ -0,0 +1,80 @@
+accuracy:
+  tables:
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8
+    #     cache_config.cpu_cache_capacity: 30
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 1
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   table_name: sift_50m_1024_128_l2
+    #   search_params:
+    #     nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+    #     top_ks: [64]
+    #     nqs: [1000]
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096
+    #     cache_config.cpu_cache_capacity: 30
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 1
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   table_name: sift_50m_1024_128_l2
+    #   search_params:
+    #     nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+    #     top_ks: [64]
+    #     nqs: [1000]
+
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_8192
+        cache_config.cpu_cache_capacity: 30
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
+        top_ks: [64]
+        nqs: [1000]
+
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096
+        cache_config.cpu_cache_capacity: 30
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      search_params:
+        nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096]
+        top_ks: [64]
+        nqs: [1000]

tests/milvus_benchmark/suites/gpu_build_performance.yaml

@@ -0,0 +1,36 @@
+build_performance:
+  tables:
+    # - 
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8h
+    #     cache_config.cpu_cache_capacity: 16
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 1
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   table_name: random_50m_1024_512_ip
+    #   index_type: ivf_sq8h
+    #   nlist: 16384
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096
+        cache_config.cpu_cache_capacity: 32
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      index_type: ivf_sq8
+      nlist: 4096

tests/milvus_benchmark/suites/gpu_build_performance_hamming50m.yaml

@@ -0,0 +1,36 @@
+build_performance:
+  tables:
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/hamming_50m_128_512_hamming_ivf
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: hamming_50m_128_512_hamming
+      index_type: ivf_flat
+      nlist: 2048
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/hamming_50m_128_512_hamming_flat
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: hamming_50m_128_512_hamming
+      index_type: flat
+      nlist: 2048

tests/milvus_benchmark/suites/gpu_build_performance_jaccard50m.yaml

@@ -0,0 +1,36 @@
+build_performance:
+  tables:
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/jaccard_50m_128_512_jaccard_ivf
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: jaccard_50m_128_512_jaccard
+      index_type: ivf_flat
+      nlist: 2048
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/jaccard_50m_128_512_jaccard_flat
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: jaccard_50m_128_512_jaccard
+      index_type: flat
+      nlist: 2048

tests/milvus_benchmark/suites/gpu_search_performance.yaml

@@ -0,0 +1,247 @@
+search_performance:
+  tables:
+    # sift_50m
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf
+        cache_config.cpu_cache_capacity: 32
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq
+    #     cache_config.cpu_cache_capacity: 32
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 200
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   table_name: sift_50m_1024_128_l2
+    #   run_count: 2
+    #   search_params:
+    #     nprobes: [8, 32]
+    #     top_ks: [1, 10, 100, 1000]
+    #     nqs: [1, 10, 100, 200, 500, 1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg
+        cache_config.cpu_cache_capacity: 50
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      run_count: 2
+      search_params:
+        nprobes: [8]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+
+    # random_50m
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_ivf
+        cache_config.cpu_cache_capacity: 110
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: random_50m_1024_512_ip
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8
+        cache_config.cpu_cache_capacity: 30
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: random_50m_1024_512_ip
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8h
+        cache_config.cpu_cache_capacity: 30
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: random_50m_1024_512_ip
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_nsg
+        cache_config.cpu_cache_capacity: 200
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: random_50m_1024_512_ip
+      run_count: 2
+      search_params:
+        nprobes: [8]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+
+    # sift_1b
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8
+        cache_config.cpu_cache_capacity: 150
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_1b_2048_128_l2
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8h
+        cache_config.cpu_cache_capacity: 150
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_1b_2048_128_l2
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq
+    #     cache_config.cpu_cache_capacity: 150
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 200
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   table_name: sift_1b_2048_128_l2
+    #   run_count: 2
+    #   search_params:
+    #     nprobes: [8, 32]
+    #     top_ks: [1, 10, 100, 1000]
+    #     nqs: [1, 10, 100, 200, 500, 1000]

tests/milvus_benchmark/suites/gpu_search_performance_hamming50m.yaml

@@ -0,0 +1,22 @@
+search_performance:
+  tables:
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/hamming_50m_128_512_hamming_ivf
+        cache_config.cpu_cache_capacity: 32
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: hamming_50m_128_512_hamming
+      run_count: 1
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 16, 64, 128, 256, 512, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]

tests/milvus_benchmark/suites/gpu_search_performance_jaccard50m.yaml

@@ -0,0 +1,22 @@
+search_performance:
+  tables:
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/jaccard_50m_128_512_jaccard_ivf
+        cache_config.cpu_cache_capacity: 32
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: jaccard_50m_128_512_jaccard
+      run_count: 1
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 16, 64, 128, 256, 512, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]

tests/milvus_benchmark/suites/gpu_search_performance_random50m.yaml

@@ -0,0 +1,82 @@
+search_performance:
+  tables:
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/random_50m_2048_512_ip_sq8
+        cache_config.cpu_cache_capacity: 110
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: random_50m_2048_512_ip
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8
+    #     cache_config.cpu_cache_capacity: 30
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 200
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   table_name: random_50m_2048_512_ip
+    #   run_count: 2
+    #   search_params:
+    #     nprobes: [8, 32]
+    #     top_ks: [1, 10, 100, 1000]
+    #     nqs: [1, 10, 100, 200, 500, 1000]
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8h
+    #     cache_config.cpu_cache_capacity: 30
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 200
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   table_name: random_50m_1024_512_ip
+    #   run_count: 2
+    #   search_params:
+    #     nprobes: [8, 32]
+    #     top_ks: [1, 10, 100, 1000]
+    #     nqs: [1, 10, 100, 200, 500, 1000]
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_nsg
+    #     cache_config.cpu_cache_capacity: 200
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 200
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   table_name: random_50m_1024_512_ip
+    #   run_count: 2
+    #   search_params:
+    #     nprobes: [8]
+    #     top_ks: [1, 10, 100, 1000]
+    #     nqs: [1, 10, 100, 200, 500, 1000]

tests/milvus_benchmark/suites/gpu_search_performance_sift1b.yaml

@@ -0,0 +1,62 @@
+search_performance:
+  tables:
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8
+        cache_config.cpu_cache_capacity: 150
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_1b_2048_128_l2
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8h
+        cache_config.cpu_cache_capacity: 150
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_1b_2048_128_l2
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq
+        cache_config.cpu_cache_capacity: 150
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_1b_2048_128_l2
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]

tests/milvus_benchmark/suites/gpu_search_performance_sift1m.yaml

@@ -0,0 +1,42 @@
+search_performance:
+  tables:
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_ivf
+        cache_config.cpu_cache_capacity: 8
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_1m_1024_128_l2
+      run_count: 1
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 16, 64, 128, 256, 512, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_sq8
+        cache_config.cpu_cache_capacity: 8
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_1m_1024_128_l2
+      run_count: 1
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 16, 64, 128, 256, 512, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]

tests/milvus_benchmark/suites/gpu_search_performance_sift50m.yaml

@@ -0,0 +1,146 @@
+search_performance:
+  tables:
+    # - 
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf
+    #     cache_config.cpu_cache_capacity: 32
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 200
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   table_name: sift_50m_1024_128_l2
+    #   run_count: 2
+    #   search_params:
+    #     nprobes: [8, 32]
+    #     top_ks: [1, 10, 100, 1000]
+    #     nqs: [1, 10, 100, 200, 500, 1000]
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8
+    #     cache_config.cpu_cache_capacity: 16
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 200
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   table_name: sift_50m_1024_128_l2
+    #   run_count: 2
+    #   search_params:
+    #     nprobes: [8, 32]
+    #     top_ks: [1, 10, 100, 1000]
+    #     nqs: [1, 10, 100, 200, 500, 1000]
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h
+    #     cache_config.cpu_cache_capacity: 16
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 200
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   table_name: sift_50m_1024_128_l2
+    #   run_count: 2
+    #   search_params:
+    #     nprobes: [8, 32]
+    #     top_ks: [1, 10, 100, 1000]
+    #     nqs: [1, 10, 100, 200, 500, 1000]
+
+    # git issue num: #626
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq
+    #     cache_config.cpu_cache_capacity: 32
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 200
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   table_name: sift_50m_1024_128_l2
+    #   run_count: 2
+    #   search_params:
+    #     nprobes: [8, 32]
+    #     top_ks: [1, 10, 100, 1000]
+    #     nqs: [1, 10, 100, 200, 500, 1000]
+    
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg
+    #     cache_config.cpu_cache_capacity: 50
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 200
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   table_name: sift_50m_1024_128_l2
+    #   run_count: 2
+    #   search_params:
+    #     nprobes: [8]
+    #     top_ks: [1, 10, 100, 1000]
+    #     nqs: [1, 10, 100, 200, 500, 1000]
+
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_8192
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096
+        cache_config.cpu_cache_capacity: 16
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      run_count: 2
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 10, 100, 1000]
+        nqs: [1, 10, 100, 200, 500, 1000]

tests/milvus_benchmark/suites/gpu_search_stability.yaml

@@ -0,0 +1,23 @@
+search_stability:
+  tables:
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8
+        cache_config.cpu_cache_capacity: 50
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+          - gpu2
+          - gpu3
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: sift_50m_1024_128_l2
+      during_time: 240
+      search_params:
+        nprobes: 1-200
+        top_ks: 1-200
+        nqs: 1-200

tests/milvus_benchmark/suites/gpu_stability_sift1m.yaml

@@ -0,0 +1,26 @@
+stability:
+  tables:
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_ivf_stability
+        cache_config.cpu_cache_capacity: 64
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_1m_1024_128_l2
+      during_time: 10
+      search_params:
+        nprobes: 1-200
+        top_ks: 1-200
+        nqs: 1-200
+      # length of insert vectors
+      insert_xb: 10000
+      # insert after search 3 times
+      insert_interval: 3

tests/milvus_benchmark/suites/gpu_stability_sift50m.yaml

@@ -0,0 +1,27 @@
+stability:
+  tables:
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h_stability
+        cache_config.cpu_cache_capacity: 64
+        cache_config.cache_insert_data: true
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_50m_1024_128_l2
+      during_time: 480
+      search_params:
+        nprobes: 1-200
+        top_ks: 1-200
+        nqs: 1-200
+      # length of insert vectors
+      insert_xb: 100000
+      # insert after search 4 times
+      insert_interval: 4

tests/milvus_benchmark/suites/insert_performance.yaml

@@ -0,0 +1,19 @@
+insert_performance:
+  tables:
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_5m_512_128_l2_ivf
+        cache_config.cpu_cache_capacity: 8
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: sift_5m_512_128_l2
+      ni_per: 100000
+      build_index: false
+      # index_type: ivf_flat
+      # nlist: 16384

tests/milvus_benchmark/suites/insert_performance_deep1b.yaml

@@ -0,0 +1,87 @@
+insert_performance:
+  tables:
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_ivf
+        cache_config.cpu_cache_capacity: 8
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: deep_1b_1024_96_ip
+      ni_per: 100000
+      build_index: false
+      # index_type: ivf_flat
+      # nlist: 16384
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_sq8
+        cache_config.cpu_cache_capacity: 8
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: deep_1b_1024_96_ip
+      ni_per: 100000
+      build_index: false
+      # index_type: ivf_sq8
+      # nlist: 16384
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_sq8h
+        cache_config.cpu_cache_capacity: 8
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: deep_1b_1024_96_ip
+      ni_per: 100000
+      build_index: false
+      # index_type: ivf_sq8h
+      # nlist: 16384
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_pq
+        cache_config.cpu_cache_capacity: 8
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: deep_1b_1024_96_ip
+      ni_per: 100000
+      build_index: false
+      # index_type: ivf_pq
+      # nlist: 16384
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_nsg
+        cache_config.cpu_cache_capacity: 8
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: deep_1b_1024_96_ip
+      ni_per: 100000
+      build_index: false
+      # index_type: nsg
+      # nlist: 16384

tests/milvus_benchmark/suites/insert_performance_hamming50m.yaml

@@ -0,0 +1,36 @@
+insert_performance:
+  tables:
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/hamming_50m_128_512_hamming_ivf
+        cache_config.cpu_cache_capacity: 8
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: hamming_50m_128_512_hamming
+      ni_per: 100000
+      build_index: false
+      # index_type: ivf_flat
+      # nlist: 16384
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/hamming_50m_128_512_hamming_flat
+        cache_config.cpu_cache_capacity: 8
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: hamming_50m_128_512_hamming
+      ni_per: 100000
+      build_index: false
+      # index_type: ivf_flat
+      # nlist: 16384

tests/milvus_benchmark/suites/insert_performance_jaccard50m.yaml

@@ -0,0 +1,36 @@
+insert_performance:
+  tables:
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/jaccard_50m_128_512_jaccard_ivf
+        cache_config.cpu_cache_capacity: 8
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: jaccard_50m_128_512_jaccard
+      ni_per: 100000
+      build_index: false
+      # index_type: ivf_flat
+      # nlist: 16384
+    # - 
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/jaccard_50m_128_512_jaccard_flat
+    #     cache_config.cpu_cache_capacity: 8
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 1
+    #     gpu_resource_config.enable: false
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #   table_name: jaccard_50m_128_512_jaccard
+    #   ni_per: 100000
+    #   build_index: false
+    #   # index_type: ivf_flat
+    #   # nlist: 16384

tests/milvus_benchmark/suites/insert_performance_sift50m.yaml

@@ -0,0 +1,87 @@
+insert_performance:
+  tables:
+    # - 
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_ip_ivf
+    #     cache_config.cpu_cache_capacity: 8
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 1
+    #     gpu_resource_config.enable: false
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #   table_name: sift_50m_1024_128_ip
+    #   ni_per: 100000
+    #   build_index: false
+    #   # index_type: ivf_flat
+    #   # nlist: 16384
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096
+        cache_config.cpu_cache_capacity: 8
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: sift_50m_1024_128_l2
+      ni_per: 100000
+      build_index: false
+      # index_type: ivf_sq8
+      # nlist: 16384
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_8192
+        cache_config.cpu_cache_capacity: 8
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      table_name: sift_50m_1024_128_l2
+      ni_per: 100000
+      build_index: false
+      # index_type: ivf_sq8h
+      # nlist: 16384
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_ip_pq
+    #     cache_config.cpu_cache_capacity: 8
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 1
+    #     gpu_resource_config.enable: false
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #   table_name: sift_50m_1024_128_ip
+    #   ni_per: 100000
+    #   build_index: false
+    #   # index_type: ivf_pq
+    #   # nlist: 16384
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_ip_nsg
+    #     cache_config.cpu_cache_capacity: 8
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 1
+    #     gpu_resource_config.enable: false
+    #     gpu_resource_config.cache_capacity: 4
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #   table_name: sift_50m_1024_128_ip
+    #   ni_per: 100000
+    #   build_index: false
+    #   # index_type: nsg
+    #   # nlist: 16384

tests/milvus_benchmark/suites/old_performance_sift50m.yaml

@@ -1,80 +0,0 @@
-performance:
-
-  # interface: search_vectors
-  query:
-    # dataset: table name you have already created
-    # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity ..
-    [
-      # {
-      #   "dataset": "sift_50m_1024_128_l2",
-      #   "index.index_types": ["mix_nsg"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [8, 32], 
-      #   "top_ks": [1, 16, 64, 128, 256, 512, 1000], 
-      #   "nqs": [1, 10, 100, 200, 500, 1000], 
-      #   "server.use_blas_threshold": 1100, 
-      #   "server.use_gpu_threshold": 100,
-      #   "server.cpu_cache_capacity": 50,
-      #   "server.gpu_cache_capacity": 6,
-      #   "server.resources": ["gpu0", "gpu1"],
-      #   "server.enable_gpu": True,
-      #   "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_nsg"
-      # },
-      # {
-      #   "dataset": "sift_50m_1024_128_l2",
-      #   "index.index_types": ["ivf_flat"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [8, 32], 
-      #   "top_ks": [1, 16, 64, 128, 256, 512, 1000], 
-      #   "nqs": [1, 10, 100, 200, 500, 1000], 
-      #   "server.use_blas_threshold": 1100, 
-      #   "server.use_gpu_threshold": 100,
-      #   "server.cpu_cache_capacity": 50,
-      #   "server.gpu_cache_capacity": 6,
-      #   "server.resources": ["gpu0", "gpu1"],
-      #   "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_ivf"
-      # },
-      # {
-      #   "dataset": "sift_50m_1024_128_l2",
-      #   "index.index_types": ["ivf_sq8"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [8, 32], 
-      #   "top_ks": [1, 16, 64, 128, 256, 512, 1000], 
-      #   "nqs": [1, 10, 100, 200, 500, 1000], 
-      #   "server.use_blas_threshold": 1100, 
-      #   "server.use_gpu_threshold": 100,
-      #   "server.cpu_cache_capacity": 50,
-      #   "server.gpu_cache_capacity": 3,
-      #   "server.resources": ["gpu0", "gpu1"],
-      #   "server.enable_gpu": True,
-      #   "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8"
-      # },
-      # {
-      #   "dataset": "sift_50m_1024_128_l2",
-      #   "index.index_types": ["ivf_sq8"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [8, 32], 
-      #   "top_ks": [1, 16, 64, 128, 256, 512, 1000], 
-      #   "nqs": [1, 10, 100, 200, 500, 1000], 
-      #   "server.use_blas_threshold": 1100, 
-      #   "server.use_gpu_threshold": 100,
-      #   "server.cpu_cache_capacity": 50,
-      #   "server.gpu_cache_capacity": 6,
-      #   "server.resources": ["gpu0", "gpu1"],
-      #   "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8"
-      # },
-      {
-        "dataset": "sift_50m_1024_128_l2",
-        "index.index_types": ["ivf_sq8"],
-        "index.nlists": [16384],
-        "nprobes": [8, 32], 
-        "top_ks": [1, 16, 64, 128, 256, 512, 1000], 
-        "nqs": [1, 10, 100, 200, 500, 1000], 
-        "server.use_blas_threshold": 1100, 
-        "server.use_gpu_threshold": 100,
-        "server.cpu_cache_capacity": 50,
-        "server.gpu_cache_capacity": 6,
-        "server.resources": ["gpu0", "gpu1"],
-        "db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2_sq8"
-      },
-    ]

tests/milvus_benchmark/suites/search_debug.yaml

@@ -0,0 +1,22 @@
+search_performance:
+  tables:
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_ivf
+        cache_config.cpu_cache_capacity: 8
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      table_name: sift_1m_1024_128_l2
+      run_count: 3
+      search_params:
+        nprobes: [8, 32]
+        top_ks: [1, 16]
+        nqs: [1, 10]

tests/milvus_benchmark/suites_accuracy.yaml

@@ -1,121 +0,0 @@
-
-accuracy:
-  # interface: search_vectors
-  query:
-    [
-      {
-        "dataset": "random_20m_1024_512_ip",
-        # index info
-        "index.index_types": ["flat", "ivf_sq8"],
-        "index.nlists": [16384],
-        "index.metric_types": ["ip"],
-        "nprobes": [1, 16, 64], 
-        "top_ks": [64], 
-        "nqs": [100], 
-        "server.cpu_cache_capacity": 100,
-        "server.resources": ["cpu", "gpu0"],
-        "db_path_prefix": "/test/milvus/db_data/random_20m_1024_512_ip",
-      },
-      # {
-      #   "dataset": "sift_50m_1024_128_l2",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8h"],
-      #   "index.nlists": [16384],
-      #   "index.metric_types": ["l2"],
-      #   "nprobes": [1, 16, 64], 
-      #   "top_ks": [64], 
-      #   "nqs": [100], 
-      #   "server.cpu_cache_capacity": 160,
-      #   "server.resources": ["cpu", "gpu0"],
-      #   "db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2",
-      #   "sift_acc": true
-      # },
-      # {
-      #   "dataset": "sift_50m_1024_128_l2",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8"],
-      #   "index.nlists": [16384],
-      #   "index.metric_types": ["l2"],
-      #   "nprobes": [1, 16, 64], 
-      #   "top_ks": [64], 
-      #   "nqs": [100], 
-      #   "server.cpu_cache_capacity": 160,
-      #   "server.resources": ["cpu", "gpu0"],
-      #   "db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2_sq8",
-      #   "sift_acc": true
-      # },
-      # {
-      #   "dataset": "sift_1b_2048_128_l2",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8h"],
-      #   "index.nlists": [16384],
-      #   "index.metric_types": ["l2"],
-      #   "nprobes": [1, 16, 64, 128], 
-      #   "top_ks": [64], 
-      #   "nqs": [100], 
-      #   "server.cpu_cache_capacity": 200,
-      #   "server.resources": ["cpu"],
-      #   "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h",
-      #   "sift_acc": true
-      # },
-      # {
-      #   "dataset": "sift_1b_2048_128_l2",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8h"],
-      #   "index.nlists": [16384],
-      #   "index.metric_types": ["l2"],
-      #   "nprobes": [1, 16, 64, 128], 
-      #   "top_ks": [64], 
-      #   "nqs": [100], 
-      #   "server.cpu_cache_capacity": 200,
-      #   "server.resources": ["cpu", "gpu0"],
-      #   "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h",
-      #   "sift_acc": true
-      # },
-      # {
-      #   "dataset": "sift_1b_2048_128_l2",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8h"],
-      #   "index.nlists": [16384],
-      #   "index.metric_types": ["l2"],
-      #   "nprobes": [1, 16, 64, 128], 
-      #   "top_ks": [64], 
-      #   "nqs": [100], 
-      #   "server.cpu_cache_capacity": 200,
-      #   "server.resources": ["cpu", "gpu0", "gpu1"],
-      #   "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h",
-      #   "sift_acc": true
-      # },
-      # {
-      #   "dataset": "sift_1m_1024_128_l2", 
-      #   "index.index_types": ["flat", "ivf_sq8"], 
-      #   "index.nlists": [16384],
-      #   "nprobes": [1, 32, 128, 256, 512], 
-      #   "nqs": 10,
-      #   "top_ks": 10, 
-      #   "server.use_blas_threshold": 1100, 
-      #   "server.cpu_cache_capacity": 16,
-      # },
-      # {
-      #   "dataset": "sift_10m_1024_128_l2", 
-      #   "index.index_types": ["flat", "ivf_sq8"], 
-      #   "index.nlists": [16384],
-      #   "nprobes": [1, 32, 128, 256, 512], 
-      #   "nqs": 10,
-      #   "top_ks": 10, 
-      #   "server.use_blas_threshold": 1100, 
-      #   "server.cpu_cache_capacity": 32,
-      # },
-      # {
-      #   "dataset": "sift_50m_1024_128_l2", 
-      #   "index.index_types": ["flat", "ivf_sq8"], 
-      #   "index.nlists": [16384],
-      #   "nprobes": [1, 32, 128, 256, 512], 
-      #   "nqs": 10,
-      #   "top_ks": 10, 
-      #   "server.use_blas_threshold": 1100, 
-      #   "server.cpu_cache_capacity": 64,
-      # }
-
-
-    ]

tests/milvus_benchmark/suites_performance.yaml

@@ -1,258 +0,0 @@
-performance:
-
-  # interface: add_vectors
-  insert:
-    # index_type: flat/ivf_flat/ivf_sq8/mix_nsg
-    [
-      # debug
-      # data_type / data_size / index_file_size / dimension
-      # data_type: random / ann_sift
-      # data_size: 10m / 1b
-      # {
-      #   "table_name": "random_50m_1024_512_ip", 
-      #   "ni_per": 100000, 
-      #   "processes": 5, # multiprocessing
-      #   "server.cpu_cache_capacity": 16,
-      #   # "server.resources": ["gpu0", "gpu1"],
-      #   "db_path_prefix": "/test/milvus/db_data"
-      # },
-      # {
-      #   "table_name": "random_5m_1024_512_ip", 
-      #   "ni_per": 100000, 
-      #   "processes": 5, # multiprocessing
-      #   "server.cpu_cache_capacity": 16,
-      #   "server.resources": ["gpu0", "gpu1"],
-      #   "db_path_prefix": "/test/milvus/db_data/random_5m_1024_512_ip"
-      # },
-      # {
-      #   "table_name": "sift_1m_50_128_l2", 
-      #   "ni_per": 100000, 
-      #   "processes": 5, # multiprocessing
-      #   # "server.cpu_cache_capacity": 16,
-      #   "db_path_prefix": "/test/milvus/db_data"
-      # },
-      # {
-      #   "table_name": "sift_1m_256_128_l2", 
-      #   "ni_per": 100000, 
-      #   "processes": 5, # multiprocessing
-      #   # "server.cpu_cache_capacity": 16,
-      #   "db_path_prefix": "/test/milvus/db_data"
-      # }
-      # {
-      #   "table_name": "sift_50m_1024_128_l2", 
-      #   "ni_per": 100000, 
-      #   "processes": 5, # multiprocessing
-      #   # "server.cpu_cache_capacity": 16,
-      # },
-      # {
-      #   "table_name": "sift_100m_1024_128_l2", 
-      #   "ni_per": 100000, 
-      #   "processes": 5, # multiprocessing
-      # },
-      # {
-      #   "table_name": "sift_1b_2048_128_l2", 
-      #   "ni_per": 100000, 
-      #   "processes": 5, # multiprocessing
-      #   "server.cpu_cache_capacity": 16,
-      # }
-    ]
-
-  # interface: search_vectors
-  query:
-    # dataset: table name you have already created
-    # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity ..
-    [
-      # {
-      #   "dataset": "sift_1b_2048_128_l2",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8h"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [8, 32], 
-      #   "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], 
-      #   "nqs": [1, 10, 100, 500, 1000], 
-      #   "processes": 1, # multiprocessing
-      #   "server.use_blas_threshold": 1100, 
-      #   "server.cpu_cache_capacity": 200,
-      #   "server.resources": ["cpu", "gpu0"],
-      #   "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h"
-      # },
-      # {
-      #   "dataset": "sift_1b_2048_128_l2",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [8, 32], 
-      #   "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], 
-      #   "nqs": [1, 10, 100, 500, 1000], 
-      #   "processes": 1, # multiprocessing
-      #   "server.use_blas_threshold": 1100, 
-      #   "server.cpu_cache_capacity": 200,
-      #   "server.resources": ["cpu", "gpu0"],
-      #   "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2"
-      # },
-      # {
-      #   "dataset": "sift_1b_2048_128_l2",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8h"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [8, 32], 
-      #   "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], 
-      #   "nqs": [1, 10, 100, 500, 1000], 
-      #   "processes": 1, # multiprocessing
-      #   "server.use_blas_threshold": 1100, 
-      #   "server.cpu_cache_capacity": 200,
-      #   "server.resources": ["cpu"],
-      #   "db_path_prefix": "/test/milvus/db_data"
-      # },
-      {
-        "dataset": "random_50m_1024_512_ip",
-        "index.index_types": ["ivf_sq8h"],
-        "index.nlists": [16384],
-        "nprobes": [8], 
-        # "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], 
-        "top_ks": [512], 
-        # "nqs": [1, 10, 100, 500, 1000], 
-        "nqs": [500], 
-        "server.use_blas_threshold": 1100, 
-        "server.cpu_cache_capacity": 150,
-        "server.gpu_cache_capacity": 6,
-        "server.resources": ["cpu", "gpu0", "gpu1"],
-        "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip"
-      },
-      # {
-      #   "dataset": "random_50m_1024_512_ip",
-      #   "index.index_types": ["ivf_sq8"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [8, 32], 
-      #   "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], 
-      #   "nqs": [1, 10, 100, 500, 1000], 
-      #   "server.use_blas_threshold": 1100, 
-      #   "server.cpu_cache_capacity": 150,
-      #   "server.resources": ["cpu", "gpu0", "gpu1"],
-      #   "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_sq8"
-      # },
-      # {
-      #   "dataset": "random_20m_1024_512_ip",
-      #   "index.index_types": ["flat"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [50], 
-      #   "top_ks": [64], 
-      #   "nqs": [10], 
-      #   "server.use_blas_threshold": 1100, 
-      #   "server.cpu_cache_capacity": 100,
-      #   "server.resources": ["cpu", "gpu0", "gpu1"],
-      #   "db_path_prefix": "/test/milvus/db_data/random_20m_1024_512_ip"
-      # },
-      # {
-      #   "dataset": "random_100m_1024_512_ip",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [8, 32], 
-      #   "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], 
-      #   "nqs": [1, 10, 100, 500, 1000], 
-      #   "processes": 1, # multiprocessing
-      #   "server.use_blas_threshold": 1100, 
-      #   "server.cpu_cache_capacity": 250,
-      #   "server.resources": ["cpu", "gpu0"],
-      #   "db_path_prefix": "/test/milvus/db_data"
-      # },
-      # {
-      #   "dataset": "random_100m_1024_512_ip",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [8, 32], 
-      #   "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], 
-      #   "nqs": [1, 10, 100, 500, 1000], 
-      #   "processes": 1, # multiprocessing
-      #   "server.use_blas_threshold": 1100, 
-      #   "server.cpu_cache_capacity": 250,
-      #   "server.resources": ["cpu"],
-      #   "db_path_prefix": "/test/milvus/db_data"
-      # },
-      # {
-      #   "dataset": "random_10m_1024_512_ip",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [1], 
-      #   "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], 
-      #   "nqs": [1, 10, 100, 500, 800], 
-      #   "processes": 1, # multiprocessing
-      #   # "server.use_blas_threshold": 1100, 
-      #   "server.cpu_cache_capacity": 16,
-      # },
-      # {
-      #   "dataset": "random_10m_1024_512_l2",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [1], 
-      #   "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], 
-      #   "nqs": [1, 10, 100, 500, 800], 
-      #   "processes": 1, # multiprocessing
-      #   "server.use_blas_threshold": 1100, 
-      #   "server.cpu_cache_capacity": 64
-      # },
-      # {
-      #   "dataset": "sift_500m_1024_128_l2",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [1], 
-      #   "top_ks": [1, 8, 16, 64, 256, 512, 1000], 
-      #   "nqs": [1, 100, 500, 800, 1000, 1500], 
-      #   # "top_ks": [256],
-      #   # "nqs": [800],
-      #   "processes": 1, # multiprocessing
-      #   # "server.use_blas_threshold": 1100, 
-      #   "server.cpu_cache_capacity": 120,
-      #   "server.resources": ["gpu0", "gpu1"],
-      #   "db_path_prefix": "/test/milvus/db_data"
-      # },
-      # {
-      #    "dataset": "sift_1b_2048_128_l2",
-      #    # index info
-      #    "index.index_types": ["ivf_sq8h"],
-      #    "index.nlists": [16384],
-      #    "nprobes": [1], 
-      #    # "top_ks": [1], 
-      #    # "nqs": [1], 
-      #    "top_ks": [256],
-      #    "nqs": [800],
-      #    "processes": 1, # multiprocessing
-      #    "server.use_blas_threshold": 1100, 
-      #    "server.cpu_cache_capacity": 110,
-      #    "server.resources": ["cpu", "gpu0"],
-      #    "db_path_prefix": "/test/milvus/db_data"
-      # },
-      # {
-      #   "dataset": "random_50m_1024_512_l2",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [1], 
-      #   "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], 
-      #   "nqs": [1, 10, 100, 500, 800], 
-      #   # "top_ks": [256],
-      #   # "nqs": [800],
-      #   "processes": 1, # multiprocessing
-      #   "server.use_blas_threshold": 1100, 
-      #   "server.cpu_cache_capacity": 128
-      # },
-    # [
-    #   {
-    #     "dataset": "sift_1m_50_128_l2",
-    #     # index info
-    #     "index.index_types": ["ivf_sq8"],
-    #     "index.nlists": [16384],
-    #     "nprobes": [1], 
-    #     "top_ks": [1], 
-    #     "nqs": [1], 
-    #     "db_path_prefix": "/test/milvus/db_data"
-    #     # "processes": 1, # multiprocessing
-    #     # "server.use_blas_threshold": 1100, 
-    #     # "server.cpu_cache_capacity": 256
-    #   }
-    ]

tests/milvus_benchmark/suites_stability.yaml

@@ -1,17 +0,0 @@
-
-stability:
-  # interface: search_vectors / add_vectors mix operation
-  query:
-    [
-      {
-        "dataset": "random_20m_1024_512_ip",
-        # "nqs": [1, 10, 100, 1000, 10000],
-        # "pds": [0.1, 0.44, 0.44, 0.02],
-        "query_process_num": 10,
-        # each 10s, do an insertion
-        # "insert_interval": 1,
-        # minutes
-        "during_time": 360,
-        "server.cpu_cache_capacity": 100
-      },
-    ]

tests/milvus_benchmark/suites_yzb.yaml

@@ -1,171 +0,0 @@
-#"server.resources": ["gpu0", "gpu1"]
-
-performance:
-  # interface: search_vectors
-  query:
-    # dataset: table name you have already created
-    # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity ..
-    [
-      # debug
-      # {
-      #   "dataset": "random_10m_1024_512_ip",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [1],
-      #   "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256],
-      #   "nqs": [1, 10, 100, 500, 800],
-      #   "processes": 1, # multiprocessing
-      #   "server.use_blas_threshold": 1100,
-      #   "server.cpu_cache_capacity": 16,
-      # },
-      # {
-      #   "dataset": "random_10m_1024_512_ip",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [1],
-      #   "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256],
-      #   "nqs": [1, 10, 100, 500, 800],
-      #   "processes": 1, # multiprocessing
-      #   "server.use_blas_threshold": 1100,
-      #   "server.cpu_cache_capacity": 16,
-      # },
-      # {
-      #   "dataset": "random_10m_1024_512_ip",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [1],
-      #   "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256],
-      #   "nqs": [1, 10, 100, 500, 800],
-      #   "processes": 1, # multiprocessing
-      #   # "server.use_blas_threshold": 1100,
-      #   "server.cpu_cache_capacity": 16,
-      # },
-      # {
-      #   "dataset": "random_10m_1024_512_ip",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [1],
-      #   "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256],
-      #   "nqs": [1, 10, 100, 500, 800],
-      #   "processes": 1, # multiprocessing
-      #   # "server.use_blas_threshold": 1100,
-      #   "server.cpu_cache_capacity": 16,
-      # },
-      # {
-      #   "dataset": "random_10m_1024_512_l2",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [1],
-      #   "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256],
-      #   "nqs": [1, 10, 100, 500, 800],
-      #   "processes": 1, # multiprocessing
-      #   "server.use_blas_threshold": 1100,
-      #   "server.cpu_cache_capacity": 64
-      # },
-#       {
-#        "dataset": "sift_50m_1024_128_l2",
-#        # index info
-#        "index.index_types": ["ivf_sq8"],
-#        "index.nlists": [16384],
-#        "nprobes": [1, 32, 128],
-#        "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256],
-#        "nqs": [1, 10, 100, 500, 800],
-#        # "top_ks": [256],
-#        # "nqs": [800],
-#        "processes": 1, # multiprocessing
-#         "server.use_blas_threshold": 1100,
-#         "server.cpu_cache_capacity": 310,
-#        "server.resources": ["gpu0", "gpu1"]
-#       },
-       {
-        "dataset": "sift_1m_1024_128_l2",
-        # index info
-        "index.index_types": ["ivf_sq8"],
-        "index.nlists": [16384],
-        "nprobes": [32],
-        "top_ks": [10],
-        "nqs": [100],
-        # "top_ks": [256],
-        # "nqs": [800],
-        "processes": 1, # multiprocessing
-         "server.use_blas_threshold": 1100,
-         "server.cpu_cache_capacity": 310,
-        "server.resources": ["cpu"]
-       },
-       {
-        "dataset": "sift_1m_1024_128_l2",
-        # index info
-        "index.index_types": ["ivf_sq8"],
-        "index.nlists": [16384],
-        "nprobes": [32],
-        "top_ks": [10],
-        "nqs": [100],
-        # "top_ks": [256],
-        # "nqs": [800],
-        "processes": 1, # multiprocessing
-         "server.use_blas_threshold": 1100,
-         "server.cpu_cache_capacity": 310,
-        "server.resources": ["gpu0"]
-       },
-       {
-        "dataset": "sift_1m_1024_128_l2",
-        # index info
-        "index.index_types": ["ivf_sq8"],
-        "index.nlists": [16384],
-        "nprobes": [32],
-        "top_ks": [10],
-        "nqs": [100],
-        # "top_ks": [256],
-        # "nqs": [800],
-        "processes": 1, # multiprocessing
-         "server.use_blas_threshold": 1100,
-         "server.cpu_cache_capacity": 310,
-        "server.resources": ["gpu0", "gpu1"]
-       },
-      # {
-      #    "dataset": "sift_1b_2048_128_l2",
-      #    # index info
-      #    "index.index_types": ["ivf_sq8"],
-      #    "index.nlists": [16384],
-      #    "nprobes": [1],
-      #    "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256],
-      #    "nqs": [1, 10, 100, 500, 800],
-      #     # "top_ks": [256],
-      #     #   "nqs": [800],
-      #    "processes": 1, # multiprocessing
-      #    "server.use_blas_threshold": 1100,
-      #    "server.cpu_cache_capacity": 310
-      # },
-#       {
-#         "dataset": "random_50m_1024_512_l2",
-#         # index info
-#         "index.index_types": ["ivf_sq8"],
-#         "index.nlists": [16384],
-#         "nprobes": [1],
-#         "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256],
-#         "nqs": [1, 10, 100, 500, 800],
-#         # "top_ks": [256],
-#         # "nqs": [800],
-#         "processes": 1, # multiprocessing
-#         "server.use_blas_threshold": 1100,
-#         "server.cpu_cache_capacity": 128,
-#         "server.resources": ["gpu0", "gpu1"]
-#       },
-      # {
-      #   "dataset": "random_100m_1024_512_ip",
-      #   # index info
-      #   "index.index_types": ["ivf_sq8"],
-      #   "index.nlists": [16384],
-      #   "nprobes": [1],
-      #   "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256],
-      #   "nqs": [1, 10, 100, 500, 800],
-      #   "processes": 1, # multiprocessing
-      #   "server.use_blas_threshold": 1100,
-      #   "server.cpu_cache_capacity": 256
-      # },
-    ]

tests/milvus_benchmark/utils.py

@@ -7,22 +7,34 @@ import os
 import sys
 import pdb
 import time
+import json
 import datetime
 import argparse
 import threading
 import logging
-import docker
-import multiprocessing
-import numpy
+import string
+import random
+# import multiprocessing
+# import numpy
 # import psutil
-from yaml import load, dump
+import h5py
+# import docker
+from yaml import full_load, dump
 import tableprint as tp
+from pprint import pprint
+from kubernetes import client, config
+
 
 logger = logging.getLogger("milvus_benchmark.utils")
+config.load_kube_config()
 
 MULTI_DB_SLAVE_PATH = "/opt/milvus/data2;/opt/milvus/data3"
 
 
+def get_unique_name():
+    return "benchmark-test-"+"".join(random.choice(string.ascii_letters + string.digits) for _ in range(8)).lower()
+
+
 def get_current_time():
     return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
 
@@ -36,11 +48,18 @@ def print_table(headers, columns, data):
     tp.table(bodys, headers)
 
 
+def get_dataset(hdf5_file_path):
+    if not os.path.exists(hdf5_file_path):
+        raise Exception("%s not existed" % hdf5_file_path)
+    dataset = h5py.File(hdf5_file_path)
+    return dataset
+
+
 def modify_config(k, v, type=None, file_path="conf/server_config.yaml", db_slave=None):
     if not os.path.isfile(file_path):
         raise Exception('File: %s not found' % file_path)
     with open(file_path) as f:
-        config_dict = load(f)
+        config_dict = full_load(f)
         f.close()
     if config_dict:
         if k.find("use_blas_threshold") != -1:
@@ -67,132 +86,258 @@ def modify_config(k, v, type=None, file_path="conf/server_config.yaml", db_slave
         raise Exception('Load file:%s error' % file_path)
 
 
-def pull_image(image):
-    registry = image.split(":")[0]
-    image_tag = image.split(":")[1]
-    client = docker.APIClient(base_url='unix://var/run/docker.sock')
-    logger.info("Start pulling image: %s" % image)
-    return client.pull(registry, image_tag)
+# update server_config.yaml
+def update_server_config(file_path, server_config):
+    if not os.path.isfile(file_path):
+        raise Exception('File: %s not found' % file_path)
+    with open(file_path) as f:
+        values_dict = full_load(f)
+        f.close()
+        for k, v in server_config.items():
+            if k.find("primary_path") != -1:
+                values_dict["db_config"]["primary_path"] = v
+            elif k.find("use_blas_threshold") != -1:
+                values_dict['engine_config']['use_blas_threshold'] = int(v)
+            elif k.find("gpu_search_threshold") != -1:
+                values_dict['engine_config']['gpu_search_threshold'] = int(v)
+            elif k.find("cpu_cache_capacity") != -1:
+                values_dict['cache_config']['cpu_cache_capacity'] = int(v)
+            elif k.find("cache_insert_data") != -1:
+                values_dict['cache_config']['cache_insert_data'] = v
+            elif k.find("enable") != -1:
+                values_dict['gpu_resource_config']['enable'] = v
+            elif k.find("gpu_cache_capacity") != -1:
+                values_dict['gpu_resource_config']['cache_capacity'] = int(v)
+            elif k.find("build_index_resources") != -1:
+                values_dict['gpu_resource_config']['build_index_resources'] = v
+            elif k.find("search_resources") != -1:  
+                values_dict['gpu_resource_config']['search_resources'] = v
+            with open(file_path, 'w') as f:
+                dump(values_dict, f, default_flow_style=False)
+            f.close()
 
 
-def run_server(image, mem_limit=None, timeout=30, test_type="local", volume_name=None, db_slave=None):
-    import colors
-
-    client = docker.from_env()
-    # if mem_limit is None:
-    #     mem_limit = psutil.virtual_memory().available
-    # logger.info('Memory limit:', mem_limit)
-    # cpu_limit = "0-%d" % (multiprocessing.cpu_count() - 1)
-    # logger.info('Running on CPUs:', cpu_limit)
-    for dir_item in ['logs', 'db']:
-        try:
-            os.mkdir(os.path.abspath(dir_item))
-        except Exception as e:
-            pass
-
-    if test_type == "local":
-        volumes = {
-            os.path.abspath('conf'):
-                {'bind': '/opt/milvus/conf', 'mode': 'ro'},
-            os.path.abspath('logs'):
-                {'bind': '/opt/milvus/logs', 'mode': 'rw'},
-            os.path.abspath('db'):
-                {'bind': '/opt/milvus/db', 'mode': 'rw'},
+# update values.yaml
+def update_values(file_path, hostname):
+    if not os.path.isfile(file_path):
+        raise Exception('File: %s not found' % file_path)
+    with open(file_path) as f:
+        values_dict = full_load(f)
+        f.close()
+    if values_dict['engine']['nodeSelector']:
+        logger.warning("nodeSelector has been set: %s" % str(values_dict['engine']['nodeSelector']))
+        return
+    # update values.yaml with the given host
+    # set limit/request cpus in resources
+    v1 = client.CoreV1Api()
+    node = v1.read_node(hostname)
+    cpus = node.status.allocatable.get("cpu")
+    # DEBUG
+    values_dict['engine']['resources'] = {
+        "limits": {
+            "cpu": str(int(cpus)-1)+".0"
+        },
+        "requests": {
+            "cpu": str(int(cpus)-2)+".0"
         }
-    elif test_type == "remote":
-        if volume_name is None:
-            raise Exception("No volume name")
-        remote_log_dir = volume_name+'/logs'
-        remote_db_dir = volume_name+'/db'
-
-        for dir_item in [remote_log_dir, remote_db_dir]:
-            if not os.path.isdir(dir_item):
-                os.makedirs(dir_item, exist_ok=True)
-        volumes = {
-            os.path.abspath('conf'):
-                {'bind': '/opt/milvus/conf', 'mode': 'ro'},
-            remote_log_dir:
-                {'bind': '/opt/milvus/logs', 'mode': 'rw'},
-            remote_db_dir:
-                {'bind': '/opt/milvus/db', 'mode': 'rw'}
+    }
+    values_dict['engine']['nodeSelector'] = {'kubernetes.io/hostname': hostname}
+    values_dict['engine']['tolerations'].append({
+        "key": "worker",
+        "operator": "Equal",
+        "value": "performance",
+        "effect": "NoSchedule"
+        })
+    # add extra volumes
+    values_dict['extraVolumes'].append({
+        'name': 'test',
+        'flexVolume': {
+            'driver': "fstab/cifs",
+            'fsType': "cifs",
+            'secretRef': {
+                'name': "cifs-test-secret"
+            },
+            'options': {
+                'networkPath': "//192.168.1.126/test",
+                'mountOptions': "vers=1.0"
+            }
         }
-        # add volumes
-        if db_slave and isinstance(db_slave, int):
-            for i in range(2, db_slave+1):
-                remote_db_dir = volume_name+'/data'+str(i)
-                if not os.path.isdir(remote_db_dir):
-                    os.makedirs(remote_db_dir, exist_ok=True)
-                volumes[remote_db_dir] = {'bind': '/opt/milvus/data'+str(i), 'mode': 'rw'}
-
-    container = client.containers.run(
-        image,
-        volumes=volumes,
-        runtime="nvidia",
-        ports={'19530/tcp': 19530, '8080/tcp': 8080},
-        # environment=["OMP_NUM_THREADS=48"],
-        # cpuset_cpus=cpu_limit,
-        # mem_limit=mem_limit,
-        # environment=[""],
-        detach=True)
-
-    def stream_logs():
-        for line in container.logs(stream=True):
-            logger.info(colors.color(line.decode().rstrip(), fg='blue'))
-
-    if sys.version_info >= (3, 0):
-        t = threading.Thread(target=stream_logs, daemon=True)
-    else:
-        t = threading.Thread(target=stream_logs)
-        t.daemon = True
-    t.start()
-
-    logger.info('Container: %s started' % container)
-    return container
-    # exit_code = container.wait(timeout=timeout)
-    # # Exit if exit code
-    # if exit_code == 0:
-    #     return container
-    # elif exit_code is not None:
-    #     print(colors.color(container.logs().decode(), fg='red'))
-    #     raise Exception('Child process raised exception %s' % str(exit_code))
-
-def restart_server(container):
-    client = docker.APIClient(base_url='unix://var/run/docker.sock')
-
-    client.restart(container.name)
-    logger.info('Container: %s restarted' % container.name)
-    return container
+    })
+    values_dict['extraVolumeMounts'].append({
+        'name': 'test',
+        'mountPath': '/test'
+    })
+    with open(file_path, 'w') as f:
+        dump(values_dict, f, default_flow_style=False)
+    f.close()
 
 
-def remove_container(container):
-    container.remove(force=True)
-    logger.info('Container: %s removed' % container)
+# deploy server
+def helm_install_server(helm_path, image_tag, image_type, name, namespace):
+    timeout = 180
+    install_cmd = "helm install --wait --timeout %d \
+            --set engine.image.tag=%s \
+            --set expose.type=clusterIP \
+            --name %s \
+            -f ci/db_backend/sqlite_%s_values.yaml \
+            -f ci/filebeat/values.yaml \
+            --namespace %s \
+            --version 0.0 ." % (timeout, image_tag, name, image_type, namespace)
+    logger.debug(install_cmd)
+    if os.system("cd %s && %s" % (helm_path, install_cmd)):
+        logger.error("Helm install failed")
+        return None
+    time.sleep(5)
+    v1 = client.CoreV1Api()
+    host = "%s-milvus-engine.%s.svc.cluster.local" % (name, namespace)
+    pod_name = None
+    pod_id = None
+    pods = v1.list_namespaced_pod(namespace)
+    for i in pods.items:
+        if i.metadata.name.find(name) != -1:
+            pod_name = i.metadata.name
+            pod_ip = i.status.pod_ip
+    logger.debug(pod_name)
+    logger.debug(pod_ip)
+    return pod_name, pod_ip
 
 
-def remove_all_containers(image):
-    client = docker.from_env()
-    try:
-        for container in client.containers.list():
-            if image in container.image.tags:
-                container.stop(timeout=30)
-                container.remove(force=True)
-    except Exception as e:
-        logger.error("Containers removed failed")
+# delete server
+def helm_del_server(name):
+    del_cmd = "helm del --purge %s" % name
+    logger.debug(del_cmd)
+    if os.system(del_cmd):
+        logger.error("Helm delete name:%s failed" % name)
+        return False
+    return True
 
 
-def container_exists(image):
-    '''
-    Check if container existed with the given image name
-    @params: image name
-    @return: container if exists
-    '''
-    res = False
-    client = docker.from_env()
-    for container in client.containers.list():
-        if image in container.image.tags:
-            # True
-            res = container
-    return res
+# def pull_image(image):
+#     registry = image.split(":")[0]
+#     image_tag = image.split(":")[1]
+#     client = docker.APIClient(base_url='unix://var/run/docker.sock')
+#     logger.info("Start pulling image: %s" % image)
+#     return client.pull(registry, image_tag)
+
+
+# def run_server(image, mem_limit=None, timeout=30, test_type="local", volume_name=None, db_slave=None):
+#     import colors
+
+#     client = docker.from_env()
+#     # if mem_limit is None:
+#     #     mem_limit = psutil.virtual_memory().available
+#     # logger.info('Memory limit:', mem_limit)
+#     # cpu_limit = "0-%d" % (multiprocessing.cpu_count() - 1)
+#     # logger.info('Running on CPUs:', cpu_limit)
+#     for dir_item in ['logs', 'db']:
+#         try:
+#             os.mkdir(os.path.abspath(dir_item))
+#         except Exception as e:
+#             pass
+
+#     if test_type == "local":
+#         volumes = {
+#             os.path.abspath('conf'):
+#                 {'bind': '/opt/milvus/conf', 'mode': 'ro'},
+#             os.path.abspath('logs'):
+#                 {'bind': '/opt/milvus/logs', 'mode': 'rw'},
+#             os.path.abspath('db'):
+#                 {'bind': '/opt/milvus/db', 'mode': 'rw'},
+#         }
+#     elif test_type == "remote":
+#         if volume_name is None:
+#             raise Exception("No volume name")
+#         remote_log_dir = volume_name+'/logs'
+#         remote_db_dir = volume_name+'/db'
+
+#         for dir_item in [remote_log_dir, remote_db_dir]:
+#             if not os.path.isdir(dir_item):
+#                 os.makedirs(dir_item, exist_ok=True)
+#         volumes = {
+#             os.path.abspath('conf'):
+#                 {'bind': '/opt/milvus/conf', 'mode': 'ro'},
+#             remote_log_dir:
+#                 {'bind': '/opt/milvus/logs', 'mode': 'rw'},
+#             remote_db_dir:
+#                 {'bind': '/opt/milvus/db', 'mode': 'rw'}
+#         }
+#         # add volumes
+#         if db_slave and isinstance(db_slave, int):
+#             for i in range(2, db_slave+1):
+#                 remote_db_dir = volume_name+'/data'+str(i)
+#                 if not os.path.isdir(remote_db_dir):
+#                     os.makedirs(remote_db_dir, exist_ok=True)
+#                 volumes[remote_db_dir] = {'bind': '/opt/milvus/data'+str(i), 'mode': 'rw'}
+
+#     container = client.containers.run(
+#         image,
+#         volumes=volumes,
+#         runtime="nvidia",
+#         ports={'19530/tcp': 19530, '8080/tcp': 8080},
+#         # environment=["OMP_NUM_THREADS=48"],
+#         # cpuset_cpus=cpu_limit,
+#         # mem_limit=mem_limit,
+#         # environment=[""],
+#         detach=True)
+
+#     def stream_logs():
+#         for line in container.logs(stream=True):
+#             logger.info(colors.color(line.decode().rstrip(), fg='blue'))
+
+#     if sys.version_info >= (3, 0):
+#         t = threading.Thread(target=stream_logs, daemon=True)
+#     else:
+#         t = threading.Thread(target=stream_logs)
+#         t.daemon = True
+#     t.start()
+
+#     logger.info('Container: %s started' % container)
+#     return container
+#     # exit_code = container.wait(timeout=timeout)
+#     # # Exit if exit code
+#     # if exit_code == 0:
+#     #     return container
+#     # elif exit_code is not None:
+#     #     print(colors.color(container.logs().decode(), fg='red'))
+#     #     raise Exception('Child process raised exception %s' % str(exit_code))
+
+# def restart_server(container):
+#     client = docker.APIClient(base_url='unix://var/run/docker.sock')
+
+#     client.restart(container.name)
+#     logger.info('Container: %s restarted' % container.name)
+#     return container
+
+
+# def remove_container(container):
+#     container.remove(force=True)
+#     logger.info('Container: %s removed' % container)
+
+
+# def remove_all_containers(image):
+#     client = docker.from_env()
+#     try:
+#         for container in client.containers.list():
+#             if image in container.image.tags:
+#                 container.stop(timeout=30)
+#                 container.remove(force=True)
+#     except Exception as e:
+#         logger.error("Containers removed failed")
+
+
+# def container_exists(image):
+#     '''
+#     Check if container existed with the given image name
+#     @params: image name
+#     @return: container if exists
+#     '''
+#     res = False
+#     client = docker.from_env()
+#     for container in client.containers.list():
+#         if image in container.image.tags:
+#             # True
+#             res = container
+#     return res
 
 
 if __name__ == '__main__':