From c5c7b97425c81faa19e60c28cee7d217d7614489 Mon Sep 17 00:00:00 2001 From: zw Date: Thu, 7 May 2020 18:09:26 +0800 Subject: [PATCH] update java tests Signed-off-by: zw --- tests/milvus-java-test/MilvusSDkJavaTest.iml | 65 ++ tests/milvus-java-test/pom.xml | 21 +- .../src/main/java/com/MainClass.java | 89 ++- .../src/main/java/com/TestAddVectors.java | 233 +++--- .../src/main/java/com/TestCollection.java | 142 ++++ .../main/java/com/TestCollectionCount.java | 77 ++ .../src/main/java/com/TestCollectionInfo.java | 70 ++ .../src/main/java/com/TestCompact.java | 60 ++ .../src/main/java/com/TestConnect.java | 20 +- .../src/main/java/com/TestDeleteVectors.java | 231 +++--- .../src/main/java/com/TestFlush.java | 102 +++ .../src/main/java/com/TestGetVectorByID.java | 91 +++ .../src/main/java/com/TestIndex.java | 373 +++++----- .../src/main/java/com/TestMix.java | 126 ++-- .../src/main/java/com/TestPS.java | 49 +- .../src/main/java/com/TestPartition.java | 156 ++++ .../src/main/java/com/TestPing.java | 4 +- .../src/main/java/com/TestSearchByIds.java | 255 +++++++ .../src/main/java/com/TestSearchVectors.java | 676 +++++------------- .../src/main/java/com/Utils.java | 79 ++ 20 files changed, 1862 insertions(+), 1057 deletions(-) create mode 100644 tests/milvus-java-test/MilvusSDkJavaTest.iml create mode 100644 tests/milvus-java-test/src/main/java/com/TestCollection.java create mode 100644 tests/milvus-java-test/src/main/java/com/TestCollectionCount.java create mode 100644 tests/milvus-java-test/src/main/java/com/TestCollectionInfo.java create mode 100644 tests/milvus-java-test/src/main/java/com/TestCompact.java create mode 100644 tests/milvus-java-test/src/main/java/com/TestFlush.java create mode 100644 tests/milvus-java-test/src/main/java/com/TestGetVectorByID.java create mode 100644 tests/milvus-java-test/src/main/java/com/TestPartition.java create mode 100644 tests/milvus-java-test/src/main/java/com/TestSearchByIds.java create mode 100644 tests/milvus-java-test/src/main/java/com/Utils.java diff --git a/tests/milvus-java-test/MilvusSDkJavaTest.iml b/tests/milvus-java-test/MilvusSDkJavaTest.iml new file mode 100644 index 0000000000..953b98ba92 --- /dev/null +++ b/tests/milvus-java-test/MilvusSDkJavaTest.iml @@ -0,0 +1,65 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/milvus-java-test/pom.xml b/tests/milvus-java-test/pom.xml index 3457842e7d..77f3e44964 100644 --- a/tests/milvus-java-test/pom.xml +++ b/tests/milvus-java-test/pom.xml @@ -28,6 +28,16 @@ + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + 3.8.1 + @@ -69,7 +79,7 @@ org.apache.commons commons-lang3 - 3.10 + 3.9 @@ -84,6 +94,13 @@ 6.14.3 + + com.alibaba + fastjson + 1.2.47 + + + junit junit @@ -99,7 +116,7 @@ io.milvus milvus-sdk-java - 0.4.1-SNAPSHOT + 0.8.0-SNAPSHOT diff --git a/tests/milvus-java-test/src/main/java/com/MainClass.java b/tests/milvus-java-test/src/main/java/com/MainClass.java index 1eaf082d58..c3f251d5da 100644 --- a/tests/milvus-java-test/src/main/java/com/MainClass.java +++ b/tests/milvus-java-test/src/main/java/com/MainClass.java @@ -15,7 +15,7 @@ import java.util.List; public class MainClass { private static String host = "127.0.0.1"; - private static String port = "19532"; + private static int port = 19530; private int index_file_size = 50; public int dimension = 128; @@ -23,7 +23,7 @@ public class MainClass { MainClass.host = host; } - public static void setPort(String port) { + public static void setPort(int port) { MainClass.port = port; } @@ -40,8 +40,8 @@ public class MainClass { .withPort(port) .build(); client.connect(connectParam); - String tableName = RandomStringUtils.randomAlphabetic(10); - return new Object[][]{{client, tableName}}; + String collectionName = RandomStringUtils.randomAlphabetic(10); + return new Object[][]{{client, collectionName}}; } @DataProvider(name="DisConnectInstance") @@ -58,16 +58,16 @@ public class MainClass { } catch (InterruptedException e) { e.printStackTrace(); } - String tableName = RandomStringUtils.randomAlphabetic(10); - return new Object[][]{{client, tableName}}; + String collectionName = RandomStringUtils.randomAlphabetic(10); + return new Object[][]{{client, collectionName}}; } - @DataProvider(name="Table") - public Object[][] provideTable() throws ConnectFailedException { - Object[][] tables = new Object[2][2]; + @DataProvider(name="Collection") + public Object[][] provideCollection() throws ConnectFailedException, InterruptedException { + Object[][] collections = new Object[2][2]; MetricType[] metricTypes = { MetricType.L2, MetricType.IP }; for (int i = 0; i < metricTypes.length; ++i) { - String tableName = metricTypes[i].toString()+"_"+RandomStringUtils.randomAlphabetic(10); + String collectionName = metricTypes[i].toString()+"_"+RandomStringUtils.randomAlphabetic(10); // Generate connection instance MilvusClient client = new MilvusGrpcClient(); ConnectParam connectParam = new ConnectParam.Builder() @@ -75,20 +75,60 @@ public class MainClass { .withPort(port) .build(); client.connect(connectParam); - TableSchema tableSchema = new TableSchema.Builder(tableName, dimension) +// List tableNames = client.showCollections().getCollectionNames(); +// for (int j = 0; j < tableNames.size(); ++j +// ) { +// client.dropCollection(tableNames.get(j)); +// } +// Thread.currentThread().sleep(2000); + CollectionMapping cm = new CollectionMapping.Builder(collectionName, dimension) .withIndexFileSize(index_file_size) .withMetricType(metricTypes[i]) .build(); - Response res = client.createTable(tableSchema); + Response res = client.createCollection(cm); if (!res.ok()) { System.out.println(res.getMessage()); - throw new SkipException("Table created failed"); + throw new SkipException("Collection created failed"); } - tables[i] = new Object[]{client, tableName}; + collections[i] = new Object[]{client, collectionName}; } - return tables; + return collections; } + @DataProvider(name="BinaryCollection") + public Object[][] provideBinaryCollection() throws ConnectFailedException, InterruptedException { + Object[][] collections = new Object[3][2]; + MetricType[] metricTypes = { MetricType.JACCARD, MetricType.HAMMING, MetricType.TANIMOTO }; + for (int i = 0; i < metricTypes.length; ++i) { + String collectionName = metricTypes[i].toString()+"_"+RandomStringUtils.randomAlphabetic(10); + // Generate connection instance + MilvusClient client = new MilvusGrpcClient(); + ConnectParam connectParam = new ConnectParam.Builder() + .withHost(host) + .withPort(port) + .build(); + client.connect(connectParam); +// List tableNames = client.showCollections().getCollectionNames(); +// for (int j = 0; j < tableNames.size(); ++j +// ) { +// client.dropCollection(tableNames.get(j)); +// } +// Thread.currentThread().sleep(2000); + CollectionMapping cm = new CollectionMapping.Builder(collectionName, dimension) + .withIndexFileSize(index_file_size) + .withMetricType(metricTypes[i]) + .build(); + Response res = client.createCollection(cm); + if (!res.ok()) { + System.out.println(res.getMessage()); + throw new SkipException("Collection created failed"); + } + collections[i] = new Object[]{client, collectionName}; + } + return collections; + } + + public static void main(String[] args) { CommandLineParser parser = new DefaultParser(); Options options = new Options(); @@ -102,7 +142,7 @@ public class MainClass { } String port = cmd.getOptionValue("port"); if (port != null) { - setPort(port); + setPort(Integer.parseInt(port)); } System.out.println("Host: "+host+", Port: "+port); } @@ -110,13 +150,6 @@ public class MainClass { System.err.println("Parsing failed. Reason: " + exp.getMessage() ); } -// TestListenerAdapter tla = new TestListenerAdapter(); -// TestNG testng = new TestNG(); -// testng.setTestClasses(new Class[] { TestPing.class }); -// testng.setTestClasses(new Class[] { TestConnect.class }); -// testng.addListener(tla); -// testng.run(); - XmlSuite suite = new XmlSuite(); suite.setName("TmpSuite"); @@ -129,9 +162,15 @@ public class MainClass { classes.add(new XmlClass("com.TestConnect")); classes.add(new XmlClass("com.TestDeleteVectors")); classes.add(new XmlClass("com.TestIndex")); + classes.add(new XmlClass("com.TestCompact")); classes.add(new XmlClass("com.TestSearchVectors")); - classes.add(new XmlClass("com.TestTable")); - classes.add(new XmlClass("com.TestTableCount")); + classes.add(new XmlClass("com.TestCollection")); + classes.add(new XmlClass("com.TestCollectionCount")); + classes.add(new XmlClass("com.TestFlush")); + classes.add(new XmlClass("com.TestPartition")); + classes.add(new XmlClass("com.TestGetVectorByID")); + classes.add(new XmlClass("com.TestCollectionInfo")); + classes.add(new XmlClass("com.TestSearchByIds")); test.setXmlClasses(classes) ; diff --git a/tests/milvus-java-test/src/main/java/com/TestAddVectors.java b/tests/milvus-java-test/src/main/java/com/TestAddVectors.java index dfb1866bb3..bff9d3d3f0 100644 --- a/tests/milvus-java-test/src/main/java/com/TestAddVectors.java +++ b/tests/milvus-java-test/src/main/java/com/TestAddVectors.java @@ -5,6 +5,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.testng.Assert; import org.testng.annotations.Test; +import java.nio.ByteBuffer; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -12,188 +13,164 @@ import java.util.stream.Stream; public class TestAddVectors { int dimension = 128; String tag = "tag"; + int nb = 8000; + List> vectors = Utils.genVectors(nb, dimension, true); + List vectorsBinary = Utils.genBinaryVectors(nb, dimension); - public List> gen_vectors(Integer nb) { - List> xb = new LinkedList<>(); - Random random = new Random(); - for (int i = 0; i < nb; ++i) { - LinkedList vector = new LinkedList<>(); - for (int j = 0; j < dimension; j++) { - vector.add(random.nextFloat()); - } - xb.add(vector); - } - return xb; - } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_add_vectors_table_not_existed(MilvusClient client, String tableName) throws InterruptedException { - int nb = 1000; - List> vectors = gen_vectors(nb); - String tableNameNew = tableName + "_"; - InsertParam insertParam = new InsertParam.Builder(tableNameNew, vectors).build(); + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_add_vectors_collection_not_existed(MilvusClient client, String collectionName) throws InterruptedException { + String collectionNameNew = collectionName + "_"; + InsertParam insertParam = new InsertParam.Builder(collectionNameNew).withFloatVectors(vectors).build(); InsertResponse res = client.insert(insertParam); assert(!res.getResponse().ok()); } @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class) - public void test_add_vectors_without_connect(MilvusClient client, String tableName) throws InterruptedException { - int nb = 100; - List> vectors = gen_vectors(nb); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + public void test_add_vectors_without_connect(MilvusClient client, String collectionName) throws InterruptedException { + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); InsertResponse res = client.insert(insertParam); assert(!res.getResponse().ok()); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_add_vectors(MilvusClient client, String tableName) throws InterruptedException { - int nb = 1000; - List> vectors = gen_vectors(nb); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_add_vectors(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); InsertResponse res = client.insert(insertParam); assert(res.getResponse().ok()); - Thread.currentThread().sleep(1000); - // Assert table row count - Assert.assertEquals(client.getTableRowCount(tableName).getTableRowCount(), nb); + Response res_flush = client.flush(collectionName); + assert(res_flush.ok()); + // Assert collection row count + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), nb); } -// @Test(dataProvider = "Table", dataProviderClass = MainClass.class) -// public void test_add_vectors_timeout(MilvusClient client, String tableName) throws InterruptedException { -// int nb = 200000; -// List> vectors = gen_vectors(nb); -// System.out.println(new Date()); -// InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withTimeout(1).build(); -// InsertResponse res = client.insert(insertParam); -// assert(!res.getResponse().ok()); -// } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_add_vectors_big_data(MilvusClient client, String tableName) throws InterruptedException { - int nb = 500000; - List> vectors = gen_vectors(nb); - System.out.println(new Date()); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); - InsertResponse res = client.insert(insertParam); - assert(res.getResponse().ok()); - } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_add_vectors_with_ids(MilvusClient client, String tableName) throws InterruptedException { - int nb = 1000; - List> vectors = gen_vectors(nb); + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_add_vectors_with_ids(MilvusClient client, String collectionName) { // Add vectors with ids List vectorIds; vectorIds = Stream.iterate(0L, n -> n) .limit(nb) .collect(Collectors.toList()); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withVectorIds(vectorIds).build(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); InsertResponse res = client.insert(insertParam); assert(res.getResponse().ok()); - Thread.currentThread().sleep(2000); - // Assert table row count - Assert.assertEquals(client.getTableRowCount(tableName).getTableRowCount(), nb); + Response res_flush = client.flush(collectionName); + assert(res_flush.ok()); + // Assert collection row count + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), nb); } - // TODO: MS-628 - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_add_vectors_with_invalid_ids(MilvusClient client, String tableName) { - int nb = 10; - List> vectors = gen_vectors(nb); + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_add_vectors_with_invalid_ids(MilvusClient client, String collectionName) { // Add vectors with ids List vectorIds; vectorIds = Stream.iterate(0L, n -> n) .limit(nb+1) .collect(Collectors.toList()); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withVectorIds(vectorIds).build(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); InsertResponse res = client.insert(insertParam); assert(!res.getResponse().ok()); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_add_vectors_with_invalid_dimension(MilvusClient client, String tableName) { - int nb = 1000; - List> vectors = gen_vectors(nb); + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_add_vectors_with_invalid_dimension(MilvusClient client, String collectionName) { vectors.get(0).add((float) 0); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); InsertResponse res = client.insert(insertParam); assert(!res.getResponse().ok()); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_add_vectors_with_invalid_vectors(MilvusClient client, String tableName) { - int nb = 1000; - List> vectors = gen_vectors(nb); + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_add_vectors_with_invalid_vectors(MilvusClient client, String collectionName) { vectors.set(0, new ArrayList<>()); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); InsertResponse res = client.insert(insertParam); assert(!res.getResponse().ok()); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_add_vectors_repeatably(MilvusClient client, String tableName) throws InterruptedException { - int nb = 100000; - int loops = 10; - List> vectors = gen_vectors(nb); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); - InsertResponse res = null; - for (int i = 0; i < loops; ++i ) { - long startTime = System.currentTimeMillis(); - res = client.insert(insertParam); - long endTime = System.currentTimeMillis(); - System.out.println("Total execution time: " + (endTime-startTime) + "ms"); - } - Thread.currentThread().sleep(1000); - // Assert table row count - Assert.assertEquals(client.getTableRowCount(tableName).getTableRowCount(), nb * loops); - } - // ----------------------------- partition cases in Insert --------------------------------- - // Add vectors into table with given tag - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_add_vectors_partition(MilvusClient client, String tableName) throws InterruptedException { - int nb = 1000; - List> vectors = gen_vectors(nb); - String partitionName = RandomStringUtils.randomAlphabetic(10); - io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build(); - Response createpResponse = client.createPartition(partition); + // Add vectors into collection with given tag + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_add_vectors_partition(MilvusClient client, String collectionName) { + Response createpResponse = client.createPartition(collectionName, tag); assert(createpResponse.ok()); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(tag).build(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).withPartitionTag(tag).build(); InsertResponse res = client.insert(insertParam); assert(res.getResponse().ok()); - Thread.currentThread().sleep(1000); - // Assert table row count - Assert.assertEquals(client.getTableRowCount(tableName).getTableRowCount(), nb); + Response res_flush = client.flush(collectionName); + assert(res_flush.ok()); + // Assert collection row count + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), nb); } - // Add vectors into table, which tag not existed - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_add_vectors_partition_tag_not_existed(MilvusClient client, String tableName) { - int nb = 1000; - String newTag = RandomStringUtils.randomAlphabetic(10); - List> vectors = gen_vectors(nb); - String partitionName = RandomStringUtils.randomAlphabetic(10); - io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build(); - Response createpResponse = client.createPartition(partition); + // Add vectors into collection, which tag not existed + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_add_vectors_partition_tag_not_existed(MilvusClient client, String collectionName) { + Response createpResponse = client.createPartition(collectionName, tag); assert(createpResponse.ok()); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(newTag).build(); + String tag = RandomStringUtils.randomAlphabetic(10); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).withPartitionTag(tag).build(); InsertResponse res = client.insert(insertParam); assert(!res.getResponse().ok()); } - // Create table, add vectors into table - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_add_vectors_partition_A(MilvusClient client, String tableName) throws InterruptedException { - int nb = 1000; - List> vectors = gen_vectors(nb); - String partitionName = RandomStringUtils.randomAlphabetic(10); - io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build(); - Response createpResponse = client.createPartition(partition); - assert(createpResponse.ok()); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + // Binary tests + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_add_vectors_partition_A_binary(MilvusClient client, String collectionName) { + Response createpResponse = client.createPartition(collectionName, tag); + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).withPartitionTag(tag).build(); InsertResponse res = client.insert(insertParam); assert(res.getResponse().ok()); - Thread.currentThread().sleep(1000); - // Assert table row count - Assert.assertEquals(client.getTableRowCount(tableName).getTableRowCount(), nb); + Response res_flush = client.flush(collectionName); + assert(res_flush.ok()); + // Assert collection row count + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), nb); + } + + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_add_vectors_binary(MilvusClient client, String collectionName) { + System.out.println(collectionName); + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + InsertResponse res = client.insert(insertParam); + assert(res.getResponse().ok()); + Response res_flush = client.flush(collectionName); + assert(res_flush.ok()); + // Assert collection row count + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), nb); + } + + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_add_vectors_with_ids_binary(MilvusClient client, String collectionName) { + // Add vectors with ids + List vectorIds; + vectorIds = Stream.iterate(0L, n -> n) + .limit(nb) + .collect(Collectors.toList()); + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + InsertResponse res = client.insert(insertParam); + assert(res.getResponse().ok()); + Response res_flush = client.flush(collectionName); + assert(res_flush.ok()); + // Assert collection row count + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), nb); + } + + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_add_vectors_with_invalid_ids_binary(MilvusClient client, String collectionName) { + // Add vectors with ids + List vectorIds; + vectorIds = Stream.iterate(0L, n -> n) + .limit(nb+1) + .collect(Collectors.toList()); + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).withVectorIds(vectorIds).build(); + InsertResponse res = client.insert(insertParam); + assert(!res.getResponse().ok()); + } + + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_add_vectors_with_invalid_dimension_binary(MilvusClient client, String collectionName) { + List vectorsBinary = Utils.genBinaryVectors(nb, dimension-1); + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + InsertResponse res = client.insert(insertParam); + assert(!res.getResponse().ok()); } } diff --git a/tests/milvus-java-test/src/main/java/com/TestCollection.java b/tests/milvus-java-test/src/main/java/com/TestCollection.java new file mode 100644 index 0000000000..b0c1be6453 --- /dev/null +++ b/tests/milvus-java-test/src/main/java/com/TestCollection.java @@ -0,0 +1,142 @@ +package com; + + +import io.milvus.client.*; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.util.List; + +public class TestCollection { + int index_file_size = 50; + int dimension = 128; + + @Test(dataProvider = "ConnectInstance", dataProviderClass = MainClass.class) + public void test_create_table(MilvusClient client, String collectionName){ + CollectionMapping tableSchema = new CollectionMapping.Builder(collectionName, dimension) + .withIndexFileSize(index_file_size) + .withMetricType(MetricType.L2) + .build(); + Response res = client.createCollection(tableSchema); + assert(res.ok()); + Assert.assertEquals(res.ok(), true); + } + + @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class) + public void test_create_table_disconnect(MilvusClient client, String collectionName){ + CollectionMapping tableSchema = new CollectionMapping.Builder(collectionName, dimension) + .withIndexFileSize(index_file_size) + .withMetricType(MetricType.L2) + .build(); + Response res = client.createCollection(tableSchema); + assert(!res.ok()); + } + + @Test(dataProvider = "ConnectInstance", dataProviderClass = MainClass.class) + public void test_create_table_repeatably(MilvusClient client, String collectionName){ + CollectionMapping tableSchema = new CollectionMapping.Builder(collectionName, dimension) + .withIndexFileSize(index_file_size) + .withMetricType(MetricType.L2) + .build(); + Response res = client.createCollection(tableSchema); + Assert.assertEquals(res.ok(), true); + Response res_new = client.createCollection(tableSchema); + Assert.assertEquals(res_new.ok(), false); + } + + @Test(dataProvider = "ConnectInstance", dataProviderClass = MainClass.class) + public void test_create_table_wrong_params(MilvusClient client, String collectionName){ + Integer dimension = 0; + CollectionMapping tableSchema = new CollectionMapping.Builder(collectionName, dimension) + .withIndexFileSize(index_file_size) + .withMetricType(MetricType.L2) + .build(); + Response res = client.createCollection(tableSchema); + System.out.println(res.toString()); + Assert.assertEquals(res.ok(), false); + } + + @Test(dataProvider = "ConnectInstance", dataProviderClass = MainClass.class) + public void test_show_tables(MilvusClient client, String collectionName){ + Integer tableNum = 10; + ShowCollectionsResponse res = null; + for (int i = 0; i < tableNum; ++i) { + String collectionNameNew = collectionName+"_"+Integer.toString(i); + CollectionMapping tableSchema = new CollectionMapping.Builder(collectionNameNew, dimension) + .withIndexFileSize(index_file_size) + .withMetricType(MetricType.L2) + .build(); + client.createCollection(tableSchema); + List collectionNames = client.showCollections().getCollectionNames(); + Assert.assertTrue(collectionNames.contains(collectionNameNew)); + } + } + + @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class) + public void test_show_tables_without_connect(MilvusClient client, String collectionName){ + ShowCollectionsResponse res = client.showCollections(); + assert(!res.getResponse().ok()); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_drop_table(MilvusClient client, String collectionName) throws InterruptedException { + Response res = client.dropCollection(collectionName); + assert(res.ok()); + Thread.currentThread().sleep(1000); + List collectionNames = client.showCollections().getCollectionNames(); + Assert.assertFalse(collectionNames.contains(collectionName)); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_drop_table_not_existed(MilvusClient client, String collectionName) { + Response res = client.dropCollection(collectionName+"_"); + assert(!res.ok()); + List collectionNames = client.showCollections().getCollectionNames(); + Assert.assertTrue(collectionNames.contains(collectionName)); + } + + @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class) + public void test_drop_table_without_connect(MilvusClient client, String collectionName) { + Response res = client.dropCollection(collectionName); + assert(!res.ok()); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_describe_table(MilvusClient client, String collectionName) { + DescribeCollectionResponse res = client.describeCollection(collectionName); + assert(res.getResponse().ok()); + CollectionMapping tableSchema = res.getCollectionMapping().get(); + Assert.assertEquals(tableSchema.getDimension(), dimension); + Assert.assertEquals(tableSchema.getCollectionName(), collectionName); + Assert.assertEquals(tableSchema.getIndexFileSize(), index_file_size); + Assert.assertEquals(tableSchema.getMetricType().name(), collectionName.substring(0,2)); + } + + @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class) + public void test_describe_table_without_connect(MilvusClient client, String collectionName) { + DescribeCollectionResponse res = client.describeCollection(collectionName); + assert(!res.getResponse().ok()); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_has_table_not_existed(MilvusClient client, String collectionName) { + HasCollectionResponse res = client.hasCollection(collectionName+"_"); + assert(res.getResponse().ok()); + Assert.assertFalse(res.hasCollection()); + } + + @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class) + public void test_has_table_without_connect(MilvusClient client, String collectionName) { + HasCollectionResponse res = client.hasCollection(collectionName); + assert(!res.getResponse().ok()); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_has_table(MilvusClient client, String collectionName) { + HasCollectionResponse res = client.hasCollection(collectionName); + assert(res.getResponse().ok()); + Assert.assertTrue(res.hasCollection()); + } + + +} diff --git a/tests/milvus-java-test/src/main/java/com/TestCollectionCount.java b/tests/milvus-java-test/src/main/java/com/TestCollectionCount.java new file mode 100644 index 0000000000..9f8b07b1b6 --- /dev/null +++ b/tests/milvus-java-test/src/main/java/com/TestCollectionCount.java @@ -0,0 +1,77 @@ +package com; + +import io.milvus.client.*; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.nio.ByteBuffer; +import java.util.List; + +public class TestCollectionCount { + int index_file_size = 50; + int dimension = 128; + int nb = 10000; + List> vectors = Utils.genVectors(nb, dimension, true); + List vectorsBinary = Utils.genBinaryVectors(nb, dimension); + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_collection_count_no_vectors(MilvusClient client, String collectionName) { + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), 0); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_collection_count_collection_not_existed(MilvusClient client, String collectionName) { + GetCollectionRowCountResponse res = client.getCollectionRowCount(collectionName+"_"); + assert(!res.getResponse().ok()); + } + + @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class) + public void test_collection_count_without_connect(MilvusClient client, String collectionName) { + GetCollectionRowCountResponse res = client.getCollectionRowCount(collectionName+"_"); + assert(!res.getResponse().ok()); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_collection_count(MilvusClient client, String collectionName) throws InterruptedException { + // Add vectors + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + client.insert(insertParam); + client.flush(collectionName); + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), nb); + } + + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_collection_count_binary(MilvusClient client, String collectionName) throws InterruptedException { + // Add vectors + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + client.insert(insertParam); + client.flush(collectionName); + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), nb); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_collection_count_multi_collections(MilvusClient client, String collectionName) throws InterruptedException { + Integer collectionNum = 10; + GetCollectionRowCountResponse res; + for (int i = 0; i < collectionNum; ++i) { + String collectionNameNew = collectionName + "_" + i; + CollectionMapping collectionSchema = new CollectionMapping.Builder(collectionNameNew, dimension) + .withIndexFileSize(index_file_size) + .withMetricType(MetricType.L2) + .build(); + client.createCollection(collectionSchema); + // Add vectors + InsertParam insertParam = new InsertParam.Builder(collectionNameNew).withFloatVectors(vectors).build(); + client.insert(insertParam); + client.flush(collectionNameNew); + } + for (int i = 0; i < collectionNum; ++i) { + String collectionNameNew = collectionName + "_" + i; + res = client.getCollectionRowCount(collectionNameNew); + Assert.assertEquals(res.getCollectionRowCount(), nb); + } + } + +} + + diff --git a/tests/milvus-java-test/src/main/java/com/TestCollectionInfo.java b/tests/milvus-java-test/src/main/java/com/TestCollectionInfo.java new file mode 100644 index 0000000000..2b17fa935b --- /dev/null +++ b/tests/milvus-java-test/src/main/java/com/TestCollectionInfo.java @@ -0,0 +1,70 @@ + +package com; + +import com.alibaba.fastjson.JSONObject; +import io.milvus.client.*; + +import org.testng.annotations.Test; + +import java.nio.ByteBuffer; +import java.util.List; + +public class TestCollectionInfo { + int dimension = 128; + int nb = 8000; + int n_list = 1024; + int default_n_list = 16384; + IndexType indexType = IndexType.IVF_SQ8; + IndexType defaultIndexType = IndexType.FLAT; + String indexParam = Utils.setIndexParam(n_list); + List> vectors = Utils.genVectors(nb, dimension, true); + List vectorsBinary = Utils.genBinaryVectors(nb, dimension); + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_get_vector_ids_after_delete_vectors(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + InsertResponse resInsert = client.insert(insertParam); + client.flush(collectionName); + List idsBefore = resInsert.getVectorIds(); + client.deleteById(collectionName, idsBefore.get(0)); + client.flush(collectionName); + Response res = client.showCollectionInfo(collectionName); + System.out.println(res.getMessage()); + JSONObject collectionInfo = Utils.getCollectionInfo(res.getMessage()); + int row_count = collectionInfo.getIntValue("row_count"); + assert(row_count == nb-1); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_get_vector_ids_after_delete_vectors_indexed(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + InsertResponse resInsert = client.insert(insertParam); + client.flush(collectionName); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + client.createIndex(index); + List idsBefore = resInsert.getVectorIds(); + client.deleteById(collectionName, idsBefore.get(0)); + client.flush(collectionName); + Response res = client.showCollectionInfo(collectionName); + System.out.println(res.getMessage()); + JSONObject collectionInfo = Utils.getCollectionInfo(res.getMessage()); + int row_count = collectionInfo.getIntValue("row_count"); + assert(row_count == nb-1); + } + + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_get_vector_ids_after_delete_vectors_binary(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + InsertResponse resInsert = client.insert(insertParam); + client.flush(collectionName); + List idsBefore = resInsert.getVectorIds(); + client.deleteById(collectionName, idsBefore.get(0)); + client.flush(collectionName); + Response res = client.showCollectionInfo(collectionName); + System.out.println(res.getMessage()); + JSONObject collectionInfo = Utils.getCollectionInfo(res.getMessage()); + int row_count = collectionInfo.getIntValue("row_count"); + assert(row_count == nb-1); + } + +} \ No newline at end of file diff --git a/tests/milvus-java-test/src/main/java/com/TestCompact.java b/tests/milvus-java-test/src/main/java/com/TestCompact.java new file mode 100644 index 0000000000..7a2ddd6622 --- /dev/null +++ b/tests/milvus-java-test/src/main/java/com/TestCompact.java @@ -0,0 +1,60 @@ +package com; + +import io.milvus.client.*; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.nio.ByteBuffer; +import java.util.List; + +public class TestCompact { + + int dimension = 128; + int nb = 8000; + List> vectors = Utils.genVectors(nb, dimension, true); + List vectorsBinary = Utils.genBinaryVectors(nb, dimension); + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_compact_after_delete(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + InsertResponse res = client.insert(insertParam); + assert(res.getResponse().ok()); + List ids = res.getVectorIds(); + client.flush(collectionName); + Response res_delete = client.deleteByIds(collectionName, ids); + assert(res_delete.ok()); + client.flush(collectionName); + Response res_compact = client.compact(collectionName); + assert(res_compact.ok()); + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), 0); + } + + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_compact_after_delete_binary(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + InsertResponse res = client.insert(insertParam); + assert(res.getResponse().ok()); + List ids = res.getVectorIds(); + client.flush(collectionName); + Response res_delete = client.deleteByIds(collectionName, ids); + assert(res_delete.ok()); + client.flush(collectionName); + Response res_compact = client.compact(collectionName); + assert(res_compact.ok()); + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), 0); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_compact_no_table(MilvusClient client, String collectionName) { + String name = ""; + Response res_compact = client.compact(name); + assert(!res_compact.ok()); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_compact_empty_table(MilvusClient client, String collectionName) { + Response res_compact = client.compact(collectionName); + assert(res_compact.ok()); + } + +} diff --git a/tests/milvus-java-test/src/main/java/com/TestConnect.java b/tests/milvus-java-test/src/main/java/com/TestConnect.java index ef89bb97ee..a89532b003 100644 --- a/tests/milvus-java-test/src/main/java/com/TestConnect.java +++ b/tests/milvus-java-test/src/main/java/com/TestConnect.java @@ -7,7 +7,7 @@ import org.testng.annotations.Test; public class TestConnect { @Test(dataProvider = "DefaultConnectArgs", dataProviderClass = MainClass.class) - public void test_connect(String host, String port) throws ConnectFailedException { + public void test_connect(String host, int port) throws ConnectFailedException { System.out.println("Host: "+host+", Port: "+port); MilvusClient client = new MilvusGrpcClient(); ConnectParam connectParam = new ConnectParam.Builder() @@ -20,7 +20,7 @@ public class TestConnect { } @Test(dataProvider = "DefaultConnectArgs", dataProviderClass = MainClass.class) - public void test_connect_repeat(String host, String port) { + public void test_connect_repeat(String host, int port) { MilvusGrpcClient client = new MilvusGrpcClient(); Response res = null; @@ -39,7 +39,7 @@ public class TestConnect { } @Test(dataProvider="InvalidConnectArgs") - public void test_connect_invalid_connect_args(String ip, String port) { + public void test_connect_invalid_connect_args(String ip, int port) { MilvusClient client = new MilvusGrpcClient(); Response res = null; try { @@ -55,29 +55,27 @@ public class TestConnect { assert(!client.isConnected()); } - // TODO: MS-615 @DataProvider(name="InvalidConnectArgs") public Object[][] generate_invalid_connect_args() { - String port = "19530"; - String ip = ""; + int port = 19530; return new Object[][]{ {"1.1.1.1", port}, {"255.255.0.0", port}, {"1.2.2", port}, {"中文", port}, - {"www.baidu.com", "100000"}, - {"127.0.0.1", "100000"}, - {"www.baidu.com", "80"}, + {"www.baidu.com", 100000}, + {"127.0.0.1", 100000}, + {"www.baidu.com", 80}, }; } @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class) - public void test_disconnect(MilvusClient client, String tableName){ + public void test_disconnect(MilvusClient client, String collectionName){ assert(!client.isConnected()); } @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class) - public void test_disconnect_repeatably(MilvusClient client, String tableName){ + public void test_disconnect_repeatably(MilvusClient client, String collectionName){ Response res = null; try { res = client.disconnect(); diff --git a/tests/milvus-java-test/src/main/java/com/TestDeleteVectors.java b/tests/milvus-java-test/src/main/java/com/TestDeleteVectors.java index d5fde0e570..f17fdf43a9 100644 --- a/tests/milvus-java-test/src/main/java/com/TestDeleteVectors.java +++ b/tests/milvus-java-test/src/main/java/com/TestDeleteVectors.java @@ -1,116 +1,149 @@ package com; -import java.util.*; +import io.milvus.client.*; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; public class TestDeleteVectors { - int index_file_size = 50; int dimension = 128; + int nb = 8000; - public List> gen_vectors(Integer nb) { - List> xb = new LinkedList<>(); - Random random = new Random(); - for (int i = 0; i < nb; ++i) { - LinkedList vector = new LinkedList<>(); - for (int j = 0; j < dimension; j++) { - vector.add(random.nextFloat()); - } - xb.add(vector); - } - return xb; + List> vectors = Utils.genVectors(nb, dimension, true); + List vectorsBinary = Utils.genBinaryVectors(nb, dimension); + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_delete_vectors(MilvusClient client, String collectionName) { + // Add vectors + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + InsertResponse res = client.insert(insertParam); + assert(res.getResponse().ok()); + List ids = res.getVectorIds(); + client.flush(collectionName); + Response res_delete = client.deleteByIds(collectionName, ids); + assert(res_delete.ok()); + client.flush(collectionName); + // Assert collection row count + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), 0); } - public static Date getDeltaDate(int delta) { - Date today = new Date(); - Calendar c = Calendar.getInstance(); - c.setTime(today); - c.add(Calendar.DAY_OF_MONTH, delta); - return c.getTime(); + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_delete_single_vector(MilvusClient client, String collectionName) { + List> del_vector = new ArrayList<>(); + del_vector.add(vectors.get(0)); + List del_ids = new ArrayList<>(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + InsertResponse res = client.insert(insertParam); + assert(res.getResponse().ok()); + List ids = res.getVectorIds(); + del_ids.add(ids.get(0)); + client.flush(collectionName); + Response res_delete = client.deleteById(collectionName, ids.get(0)); + assert(res_delete.ok()); + client.flush(collectionName); + // Assert collection row count + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), nb - 1); + GetVectorsByIdsResponse res_get = client.getVectorsByIds(collectionName, del_ids); + assert(res_get.getResponse().ok()); + assert(res_get.getFloatVectors().get(0).size() == 0); } -// @Test(dataProvider = "Table", dataProviderClass = MainClass.class) -// public void test_delete_vectors(MilvusClient client, String tableName) throws InterruptedException { -// int nb = 10000; -// List> vectors = gen_vectors(nb); -// // Add vectors -// InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); -// InsertResponse res = client.insert(insertParam); -// assert(res.getResponse().ok()); -// Thread.sleep(1000); -// DateRange dateRange = new DateRange(getDeltaDate(-1), getDeltaDate(1)); -// DeleteByRangeParam param = new DeleteByRangeParam.Builder(dateRange, tableName).build(); -// Response res_delete = client.deleteByRange(param); -// assert(res_delete.ok()); -// Thread.sleep(1000); -// // Assert table row count -// Assert.assertEquals(client.getTableRowCount(tableParam).getTableRowCount(), 0); -// } + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_delete_vectors_collection_not_existed(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + InsertResponse res = client.insert(insertParam); + assert(res.getResponse().ok()); + client.flush(collectionName); + List ids = res.getVectorIds(); + Response res_delete = client.deleteByIds(collectionName + "_not_existed", ids); + assert(!res_delete.ok()); + } -// @Test(dataProvider = "Table", dataProviderClass = MainClass.class) -// public void test_delete_vectors_table_not_existed(MilvusClient client, String tableName) throws InterruptedException { -// String tableNameNew = tableName + "_"; -// DateRange dateRange = new DateRange(getDeltaDate(-1), getDeltaDate(1)); -// DeleteByRangeParam param = new DeleteByRangeParam.Builder(dateRange, tableNameNew).build(); -// Response res_delete = client.deleteByRange(param); -// assert(!res_delete.ok()); -// } + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_delete_vector_id_not_existed(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + InsertResponse res = client.insert(insertParam); + assert(res.getResponse().ok()); + List ids = new ArrayList(); + ids.add((long)123456); + ids.add((long)1234561); + client.flush(collectionName); + Response res_delete = client.deleteByIds(collectionName, ids); + assert(res_delete.ok()); + client.flush(collectionName); + // Assert collection row count + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), nb); + } -// @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class) -// public void test_delete_vectors_without_connect(MilvusClient client, String tableName) throws InterruptedException { -// DateRange dateRange = new DateRange(getDeltaDate(-1), getDeltaDate(1)); -// DeleteByRangeParam param = new DeleteByRangeParam.Builder(dateRange, tableName).build(); -// Response res_delete = client.deleteByRange(param); -// assert(!res_delete.ok()); -// } -// -// @Test(dataProvider = "Table", dataProviderClass = MainClass.class) -// public void test_delete_vectors_table_empty(MilvusClient client, String tableName) throws InterruptedException { -// DateRange dateRange = new DateRange(getDeltaDate(-1), getDeltaDate(1)); -// DeleteByRangeParam param = new DeleteByRangeParam.Builder(dateRange, tableName).build(); -// Response res_delete = client.deleteByRange(param); -// assert(res_delete.ok()); -// } -// @Test(dataProvider = "Table", dataProviderClass = MainClass.class) -// public void test_delete_vectors_invalid_date_range(MilvusClient client, String tableName) throws InterruptedException { -// int nb = 100; -// List> vectors = gen_vectors(nb); -// // Add vectors -// InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); -// InsertResponse res = client.insert(insertParam); -// assert(res.getResponse().ok()); -// Thread.sleep(1000); -// DateRange dateRange = new DateRange(getDeltaDate(1), getDeltaDate(0)); -// DeleteByRangeParam param = new DeleteByRangeParam.Builder(dateRange, tableName).build(); -// Response res_delete = client.deleteByRange(param); -// assert(!res_delete.ok()); -// } + // Below tests binary vectors + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_delete_vectors_binary(MilvusClient client, String collectionName) { + // Add vectors + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + InsertResponse res = client.insert(insertParam); + assert(res.getResponse().ok()); + List ids = res.getVectorIds(); + client.flush(collectionName); + Response res_delete = client.deleteByIds(collectionName, ids); + assert(res_delete.ok()); + client.flush(collectionName); + // Assert collection row count + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), 0); + } -// @Test(dataProvider = "Table", dataProviderClass = MainClass.class) -// public void test_delete_vectors_invalid_date_range_1(MilvusClient client, String tableName) throws InterruptedException { -// int nb = 100; -// List> vectors = gen_vectors(nb); -// InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); -// InsertResponse res = client.insert(insertParam); -// assert(res.getResponse().ok()); -// DateRange dateRange = new DateRange(getDeltaDate(2), getDeltaDate(-1)); -// DeleteByRangeParam param = new DeleteByRangeParam.Builder(dateRange, tableName).build(); -// Response res_delete = client.deleteByRange(param); -// assert(!res_delete.ok()); -// } + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_delete_single_vector_binary(MilvusClient client, String collectionName) { + List del_vector = new ArrayList<>(); + del_vector.add(vectorsBinary.get(0)); + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + InsertResponse res = client.insert(insertParam); + assert(res.getResponse().ok()); + List ids = res.getVectorIds(); + client.flush(collectionName); + Response res_delete = client.deleteById(collectionName, ids.get(0)); + assert(res_delete.ok()); + client.flush(collectionName); + // Assert collection row count + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), nb - 1); + // Cannot search for the vector + SearchParam searchParam = new SearchParam.Builder(collectionName) + .withBinaryVectors(del_vector) + .withTopK(1) + .withParamsInJson("{\"nprobe\": 20}") + .build(); + SearchResponse res_search = client.search(searchParam); + assert(res_search.getResultIdsList().size() == 1); + } -// @Test(dataProvider = "Table", dataProviderClass = MainClass.class) -// public void test_delete_vectors_no_result(MilvusClient client, String tableName) throws InterruptedException { -// int nb = 100; -// List> vectors = gen_vectors(nb); -// InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); -// InsertResponse res = client.insert(insertParam); -// assert(res.getResponse().ok()); -// Thread.sleep(1000); -// DateRange dateRange = new DateRange(getDeltaDate(-3), getDeltaDate(-2)); -// DeleteByRangeParam param = new DeleteByRangeParam.Builder(dateRange, tableName).build(); -// Response res_delete = client.deleteByRange(param); -// assert(res_delete.ok()); -// Assert.assertEquals(client.getTableRowCount(tableParam).getTableRowCount(), nb); -// } + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_delete_vectors_collection_not_existed_binary(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + InsertResponse res = client.insert(insertParam); + assert(res.getResponse().ok()); + List ids = res.getVectorIds(); + client.flush(collectionName); + Response res_delete = client.deleteByIds(collectionName + "_not_existed", ids); + assert(!res_delete.ok()); + } + + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_delete_vector_id_not_existed_binary(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + InsertResponse res = client.insert(insertParam); + assert(res.getResponse().ok()); + List ids = new ArrayList(); + ids.add((long)123456); + ids.add((long)1234561); + client.flush(collectionName); + Response res_delete = client.deleteByIds(collectionName, ids); + assert(res_delete.ok()); + client.flush(collectionName); + // Assert collection row count + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), nb); + } } diff --git a/tests/milvus-java-test/src/main/java/com/TestFlush.java b/tests/milvus-java-test/src/main/java/com/TestFlush.java new file mode 100644 index 0000000000..5255f3b6f2 --- /dev/null +++ b/tests/milvus-java-test/src/main/java/com/TestFlush.java @@ -0,0 +1,102 @@ +package com; + +import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.ExecutionException; +import io.milvus.client.*; + +import org.apache.commons.lang3.RandomStringUtils; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class TestFlush { + int index_file_size = 50; + int dimension = 128; + int nb = 8000; + + List> vectors = Utils.genVectors(nb, dimension, true); + List vectorsBinary = Utils.genBinaryVectors(nb, dimension); + + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_flush_collection_not_existed(MilvusClient client, String collectionName) { + String newCollection = "not_existed"; + Response res = client.flush(newCollection); + assert(!res.ok()); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_flush_empty_collection(MilvusClient client, String collectionName) { + Response res = client.flush(collectionName); + assert(res.ok()); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_add_collections_flush(MilvusClient client, String collectionName) { + List names = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + names.add(RandomStringUtils.randomAlphabetic(10)); + CollectionMapping tableSchema = new CollectionMapping.Builder(names.get(i), dimension) + .withIndexFileSize(index_file_size) + .withMetricType(MetricType.IP) + .build(); + client.createCollection(tableSchema); + InsertParam insertParam = new InsertParam.Builder(names.get(i)).withFloatVectors(vectors).build(); + client.insert(insertParam); + System.out.println("Table " + names.get(i) + " created."); + } + Response res = client.flush(names); + assert(res.ok()); + for (int i = 0; i < 10; i++) { + // check row count + Assert.assertEquals(client.getCollectionRowCount(names.get(i)).getCollectionRowCount(), nb); + } + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_add_collections_flush_async(MilvusClient client, String collectionName) throws ExecutionException, InterruptedException { + List names = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + names.add(RandomStringUtils.randomAlphabetic(10)); + CollectionMapping tableSchema = new CollectionMapping.Builder(names.get(i), dimension) + .withIndexFileSize(index_file_size) + .withMetricType(MetricType.IP) + .build(); + client.createCollection(tableSchema); + InsertParam insertParam = new InsertParam.Builder(names.get(i)).withFloatVectors(vectors).build(); + client.insert(insertParam); + System.out.println("Collection " + names.get(i) + " created."); + } + ListenableFuture flushResponseFuture = client.flushAsync(names); + flushResponseFuture.get(); + for (int i = 0; i < 100; i++) { + // check row count + Assert.assertEquals(client.getCollectionRowCount(names.get(i)).getCollectionRowCount(), nb); + } + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_add_flush_multiple_times(MilvusClient client, String collectionName) { + for (int i = 0; i < 10; i++) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + client.insert(insertParam); + Response res = client.flush(collectionName); + assert(res.ok()); + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), nb * (i+1)); + } + } + + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_add_flush_multiple_times_binary(MilvusClient client, String collectionName) { + for (int i = 0; i < 10; i++) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + client.insert(insertParam); + Response res = client.flush(collectionName); + assert(res.ok()); + Assert.assertEquals(client.getCollectionRowCount(collectionName).getCollectionRowCount(), nb * (i+1)); + } + } +} \ No newline at end of file diff --git a/tests/milvus-java-test/src/main/java/com/TestGetVectorByID.java b/tests/milvus-java-test/src/main/java/com/TestGetVectorByID.java new file mode 100644 index 0000000000..efa80bdd64 --- /dev/null +++ b/tests/milvus-java-test/src/main/java/com/TestGetVectorByID.java @@ -0,0 +1,91 @@ +package com; + +import io.milvus.client.*; +import org.testng.annotations.Test; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class TestGetVectorByID { + int dimension = 128; + int nb = 8000; + public List get_ids = Utils.toListIds(1111); + List> vectors = Utils.genVectors(nb, dimension, true); + List vectorsBinary = Utils.genBinaryVectors(nb, dimension); + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_get_vector_by_id_valid(MilvusClient client, String collectionName) { + int get_length = 100; + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + InsertResponse resInsert = client.insert(insertParam); + List ids = resInsert.getVectorIds(); + client.flush(collectionName); + GetVectorsByIdsResponse res = client.getVectorsByIds(collectionName, ids.subList(0, get_length)); + assert (res.getResponse().ok()); + for (int i = 0; i < get_length; i++) { + assert (res.getFloatVectors().get(i).equals(vectors.get(i))); + } + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_get_vector_by_id_after_delete(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + InsertResponse resInsert = client.insert(insertParam); + List ids = resInsert.getVectorIds(); + Response res_delete = client.deleteById(collectionName, ids.get(0)); + assert(res_delete.ok()); + client.flush(collectionName); + GetVectorsByIdsResponse res = client.getVectorsByIds(collectionName, ids.subList(0, 1)); + assert (res.getResponse().ok()); + assert (res.getFloatVectors().get(0).size() == 0); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_get_vector_by_id_collection_name_not_existed(MilvusClient client, String collectionName) { + String newCollection = "not_existed"; + GetVectorsByIdsResponse res = client.getVectorsByIds(newCollection, get_ids); + assert(!res.getResponse().ok()); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_get_vector_id_not_existed(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + client.insert(insertParam); + client.flush(collectionName); + GetVectorsByIdsResponse res = client.getVectorsByIds(collectionName, get_ids); + assert (res.getFloatVectors().get(0).size() == 0); + } + + // Binary tests + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_get_vector_by_id_valid_binary(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + InsertResponse resInsert = client.insert(insertParam); + List ids = resInsert.getVectorIds(); + client.flush(collectionName); + GetVectorsByIdsResponse res = client.getVectorsByIds(collectionName, ids.subList(0, 1)); + assert res.getBinaryVectors().get(0).equals(vectorsBinary.get(0).rewind()); + } + + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_get_vector_by_id_after_delete_binary(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + InsertResponse resInsert = client.insert(insertParam); + List ids = resInsert.getVectorIds(); + Response res_delete = client.deleteById(collectionName, ids.get(0)); + assert(res_delete.ok()); + client.flush(collectionName); + GetVectorsByIdsResponse res = client.getVectorsByIds(collectionName, ids.subList(0, 1)); + assert (res.getFloatVectors().get(0).size() == 0); + } + + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_get_vector_id_not_existed_binary(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + client.insert(insertParam); + client.flush(collectionName); + GetVectorsByIdsResponse res = client.getVectorsByIds(collectionName, get_ids); + assert (res.getFloatVectors().get(0).size() == 0); + } +} \ No newline at end of file diff --git a/tests/milvus-java-test/src/main/java/com/TestIndex.java b/tests/milvus-java-test/src/main/java/com/TestIndex.java index a87befd97a..515bca2709 100644 --- a/tests/milvus-java-test/src/main/java/com/TestIndex.java +++ b/tests/milvus-java-test/src/main/java/com/TestIndex.java @@ -4,321 +4,292 @@ import io.milvus.client.*; import org.testng.Assert; import org.testng.annotations.Test; -import java.util.LinkedList; +import java.nio.ByteBuffer; import java.util.List; -import java.util.Random; +import java.util.logging.Logger; public class TestIndex { - int index_file_size = 10; int dimension = 128; int n_list = 1024; int default_n_list = 16384; int nb = 100000; IndexType indexType = IndexType.IVF_SQ8; IndexType defaultIndexType = IndexType.FLAT; + String indexParam = Utils.setIndexParam(n_list); - public List> gen_vectors(Integer nb) { - List> xb = new LinkedList<>(); - Random random = new Random(); - for (int i = 0; i < nb; ++i) { - LinkedList vector = new LinkedList<>(); - for (int j = 0; j < dimension; j++) { - vector.add(random.nextFloat()); - } - xb.add(vector); - } - return xb; - } + List> vectors = Utils.genVectors(nb, dimension, true); + List vectorsBinary = Utils.genBinaryVectors(nb, dimension); - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_create_index(MilvusClient client, String tableName) throws InterruptedException { - List> vectors = gen_vectors(nb); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_create_index(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - Response res_create = client.createIndex(createIndexParam); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + Response res_create = client.createIndex(index); assert(res_create.ok()); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_create_index_repeatably(MilvusClient client, String tableName) throws InterruptedException { - List> vectors = gen_vectors(nb); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_create_index_binary(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - Response res_create = client.createIndex(createIndexParam); - res_create = client.createIndex(createIndexParam); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + Response res_create = client.createIndex(index); assert(res_create.ok()); - DescribeIndexResponse res = client.describeIndex(tableName); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_create_index_repeatably(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + client.insert(insertParam); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + Response res_create = client.createIndex(index); + assert(res_create.ok()); + DescribeIndexResponse res = client.describeIndex(collectionName); assert(res.getResponse().ok()); Index index1 = res.getIndex().get(); - Assert.assertEquals(index1.getNList(), n_list); + Assert.assertEquals(Utils.getIndexParamValue(index1.getParamsInJson(), "nlist"), n_list); Assert.assertEquals(index1.getIndexType(), indexType); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_create_index_FLAT(MilvusClient client, String tableName) throws InterruptedException { + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_create_index_FLAT(MilvusClient client, String collectionName) { IndexType indexType = IndexType.FLAT; - List> vectors = gen_vectors(nb); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + Logger.getLogger("a").info("start insert"); client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - Response res_create = client.createIndex(createIndexParam); + Logger.getLogger("a").info("end insert"); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + Response res_create = client.createIndex(index); assert(res_create.ok()); - DescribeIndexResponse res = client.describeIndex(tableName); + Logger.getLogger("a").info("end create"); + DescribeIndexResponse res = client.describeIndex(collectionName); assert(res.getResponse().ok()); Index index1 = res.getIndex().get(); Assert.assertEquals(index1.getIndexType(), indexType); } -// @Test(dataProvider = "Table", dataProviderClass = MainClass.class) -// public void test_create_index_FLAT_timeout(MilvusClient client, String tableName) throws InterruptedException { -// int nb = 500000; -// IndexType indexType = IndexType.IVF_SQ8; -// List> vectors = gen_vectors(nb); -// InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); -// client.insert(insertParam); -// Index index = new Index.Builder().withIndexType(indexType) -// .withNList(n_list) -// .build(); -// System.out.println(new Date()); -// CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).withTimeout(1).build(); -// Response res_create = client.createIndex(createIndexParam); -// assert(!res_create.ok()); -// } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_create_index_IVFLAT(MilvusClient client, String tableName) throws InterruptedException { + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_create_index_IVFLAT(MilvusClient client, String collectionName) { IndexType indexType = IndexType.IVFLAT; - List> vectors = gen_vectors(nb); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - Response res_create = client.createIndex(createIndexParam); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + Response res_create = client.createIndex(index); assert(res_create.ok()); - DescribeIndexResponse res = client.describeIndex(tableName); + DescribeIndexResponse res = client.describeIndex(collectionName); assert(res.getResponse().ok()); Index index1 = res.getIndex().get(); Assert.assertEquals(index1.getIndexType(), indexType); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_create_index_IVFSQ8(MilvusClient client, String tableName) throws InterruptedException { + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_create_index_IVFSQ8(MilvusClient client, String collectionName) { IndexType indexType = IndexType.IVF_SQ8; - List> vectors = gen_vectors(nb); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - Response res_create = client.createIndex(createIndexParam); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + Response res_create = client.createIndex(index); assert(res_create.ok()); - DescribeIndexResponse res = client.describeIndex(tableName); + DescribeIndexResponse res = client.describeIndex(collectionName); assert(res.getResponse().ok()); Index index1 = res.getIndex().get(); Assert.assertEquals(index1.getIndexType(), indexType); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_create_index_IVFSQ8H(MilvusClient client, String tableName) throws InterruptedException { + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_create_index_IVFSQ8H(MilvusClient client, String collectionName) { IndexType indexType = IndexType.IVF_SQ8_H; - List> vectors = gen_vectors(nb); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - Response res_create = client.createIndex(createIndexParam); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + Response res_create = client.createIndex(index); assert(res_create.ok()); - DescribeIndexResponse res = client.describeIndex(tableName); + DescribeIndexResponse res = client.describeIndex(collectionName); assert(res.getResponse().ok()); Index index1 = res.getIndex().get(); Assert.assertEquals(index1.getIndexType(), indexType); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_create_index_with_no_vector(MilvusClient client, String tableName) { - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - Response res_create = client.createIndex(createIndexParam); + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_create_index_with_no_vector(MilvusClient client, String collectionName) { + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + Response res_create = client.createIndex(index); assert(res_create.ok()); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_create_index_table_not_existed(MilvusClient client, String tableName) throws InterruptedException { - String tableNameNew = tableName + "_"; - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableNameNew).withIndex(index).build(); - Response res_create = client.createIndex(createIndexParam); + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_create_index_table_not_existed(MilvusClient client, String collectionName) { + String collectionNameNew = collectionName + "_"; + Index index = new Index.Builder(collectionNameNew, indexType).withParamsInJson(indexParam).build(); + Response res_create = client.createIndex(index); assert(!res_create.ok()); } @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class) - public void test_create_index_without_connect(MilvusClient client, String tableName) throws InterruptedException { - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - Response res_create = client.createIndex(createIndexParam); + public void test_create_index_without_connect(MilvusClient client, String collectionName) { + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + Response res_create = client.createIndex(index); assert(!res_create.ok()); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_create_index_invalid_n_list(MilvusClient client, String tableName) throws InterruptedException { + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_create_index_invalid_n_list(MilvusClient client, String collectionName) { int n_list = 0; - List> vectors = gen_vectors(nb); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); - client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); + String indexParamNew = Utils.setIndexParam(n_list); + Index createIndexParam = new Index.Builder(collectionName, indexType).withParamsInJson(indexParamNew).build(); Response res_create = client.createIndex(createIndexParam); assert(!res_create.ok()); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_describe_index(MilvusClient client, String tableName) throws InterruptedException { - List> vectors = gen_vectors(nb); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); - client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_create_index_invalid_n_list_binary(MilvusClient client, String collectionName) { + int n_list = 0; + String indexParamNew = Utils.setIndexParam(n_list); + Index createIndexParam = new Index.Builder(collectionName, indexType).withParamsInJson(indexParamNew).build(); Response res_create = client.createIndex(createIndexParam); + assert(!res_create.ok()); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_describe_index(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + client.insert(insertParam); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + Response res_create = client.createIndex(index); assert(res_create.ok()); - DescribeIndexResponse res = client.describeIndex(tableName); + DescribeIndexResponse res = client.describeIndex(collectionName); assert(res.getResponse().ok()); Index index1 = res.getIndex().get(); - Assert.assertEquals(index1.getNList(), n_list); + Assert.assertEquals(Utils.getIndexParamValue(index1.getParamsInJson(), "nlist"), n_list); Assert.assertEquals(index1.getIndexType(), indexType); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_alter_index(MilvusClient client, String tableName) throws InterruptedException { - List> vectors = gen_vectors(nb); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_describe_index_binary(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - Response res_create = client.createIndex(createIndexParam); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + Response res_create = client.createIndex(index); + assert(!res_create.ok()); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_after_index(MilvusClient client, String collectionName) { + int n_list = 1025; + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + client.insert(insertParam); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + Response res_create = client.createIndex(index); assert(res_create.ok()); // Create another index IndexType indexTypeNew = IndexType.IVFLAT; - int n_list_new = n_list + 1; - Index index_new = new Index.Builder().withIndexType(indexTypeNew) - .withNList(n_list_new) - .build(); - CreateIndexParam createIndexParamNew = new CreateIndexParam.Builder(tableName).withIndex(index_new).build(); - Response res_create_new = client.createIndex(createIndexParamNew); - assert(res_create_new.ok()); - DescribeIndexResponse res = client.describeIndex(tableName); + String indexParam = Utils.setIndexParam(n_list); + Index indexNew = new Index.Builder(collectionName, indexTypeNew).withParamsInJson(indexParam).build(); + Response resNew = client.createIndex(indexNew); + assert(resNew.ok()); + DescribeIndexResponse res = client.describeIndex(collectionName); assert(res_create.ok()); Index index1 = res.getIndex().get(); - Assert.assertEquals(index1.getNList(), n_list_new); + Assert.assertEquals(Utils.getIndexParamValue(index1.getParamsInJson(), "nlist"), n_list); Assert.assertEquals(index1.getIndexType(), indexTypeNew); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_describe_index_table_not_existed(MilvusClient client, String tableName) throws InterruptedException { - String tableNameNew = tableName + "_"; - DescribeIndexResponse res = client.describeIndex(tableNameNew); + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_describe_index_table_not_existed(MilvusClient client, String collectionName) { + String collectionNameNew = collectionName + "_"; + DescribeIndexResponse res = client.describeIndex(collectionNameNew); assert(!res.getResponse().ok()); } @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class) - public void test_describe_index_without_connect(MilvusClient client, String tableName) throws InterruptedException { - DescribeIndexResponse res = client.describeIndex(tableName); + public void test_describe_index_without_connect(MilvusClient client, String collectionName) { + DescribeIndexResponse res = client.describeIndex(collectionName); assert(!res.getResponse().ok()); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_drop_index(MilvusClient client, String tableName) throws InterruptedException { - List> vectors = gen_vectors(nb); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_drop_index(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); client.insert(insertParam); - Index index = new Index.Builder().withIndexType(defaultIndexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - Response res_create = client.createIndex(createIndexParam); - assert(res_create.ok()); - Response res_drop = client.dropIndex(tableName); + Index indexNew = new Index.Builder(collectionName, defaultIndexType).withParamsInJson(indexParam).build(); + Response res = client.createIndex(indexNew); + assert(res.ok()); + Response res_drop = client.dropIndex(collectionName); assert(res_drop.ok()); - DescribeIndexResponse res = client.describeIndex(tableName); - assert(res.getResponse().ok()); - Index index1 = res.getIndex().get(); - Assert.assertEquals(index1.getNList(), default_n_list); + DescribeIndexResponse res2 = client.describeIndex(collectionName); + assert(res2.getResponse().ok()); + Index index1 = res2.getIndex().get(); + Assert.assertEquals(Utils.getIndexParamValue(index1.getParamsInJson(), "nlist"), 0); Assert.assertEquals(index1.getIndexType(), defaultIndexType); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_drop_index_repeatably(MilvusClient client, String tableName) throws InterruptedException { - List> vectors = gen_vectors(nb); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_drop_index_binary(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); client.insert(insertParam); - Index index = new Index.Builder().withIndexType(defaultIndexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - Response res_create = client.createIndex(createIndexParam); - assert(res_create.ok()); - Response res_drop = client.dropIndex(tableName); - res_drop = client.dropIndex(tableName); + Index indexNew = new Index.Builder(collectionName, defaultIndexType).withParamsInJson(indexParam).build(); + Response res = client.createIndex(indexNew); + assert(res.ok()); + Response res_drop = client.dropIndex(collectionName); assert(res_drop.ok()); - DescribeIndexResponse res = client.describeIndex(tableName); - assert(res.getResponse().ok()); - Index index1 = res.getIndex().get(); - Assert.assertEquals(index1.getNList(), default_n_list); + DescribeIndexResponse res2 = client.describeIndex(collectionName); + assert(res2.getResponse().ok()); + Index index1 = res2.getIndex().get(); + Assert.assertEquals(Utils.getIndexParamValue(index1.getParamsInJson(), "nlist"), 0); Assert.assertEquals(index1.getIndexType(), defaultIndexType); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_drop_index_table_not_existed(MilvusClient client, String tableName) throws InterruptedException { - String tableNameNew = tableName + "_"; - Response res_drop = client.dropIndex(tableNameNew); + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_drop_index_repeatably(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + client.insert(insertParam); + Index index = new Index.Builder(collectionName, defaultIndexType).withParamsInJson(indexParam).build(); + Response res = client.createIndex(index); + assert(res.ok()); + Response res_drop = client.dropIndex(collectionName); + assert(res_drop.ok()); + DescribeIndexResponse res_desc = client.describeIndex(collectionName); + assert(res_desc.getResponse().ok()); + Index index1 = res_desc.getIndex().get(); + Assert.assertEquals(Utils.getIndexParamValue(index1.getParamsInJson(), "nlist"), 0); + Assert.assertEquals(index1.getIndexType(), defaultIndexType); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_drop_index_table_not_existed(MilvusClient client, String collectionName) { + String collectionNameNew = collectionName + "_"; + Response res_drop = client.dropIndex(collectionNameNew); assert(!res_drop.ok()); } @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class) - public void test_drop_index_without_connect(MilvusClient client, String tableName) throws InterruptedException { - Response res_drop = client.dropIndex(tableName); + public void test_drop_index_without_connect(MilvusClient client, String collectionName) { + Response res_drop = client.dropIndex(collectionName); assert(!res_drop.ok()); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_drop_index_no_index_created(MilvusClient client, String tableName) throws InterruptedException { - List> vectors = gen_vectors(nb); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_drop_index_no_index_created(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); client.insert(insertParam); - Response res_drop = client.dropIndex(tableName); + Response res_drop = client.dropIndex(collectionName); assert(res_drop.ok()); - DescribeIndexResponse res = client.describeIndex(tableName); + DescribeIndexResponse res = client.describeIndex(collectionName); assert(res.getResponse().ok()); Index index1 = res.getIndex().get(); - Assert.assertEquals(index1.getNList(), default_n_list); Assert.assertEquals(index1.getIndexType(), defaultIndexType); } + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_drop_index_no_index_created_binary(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + client.insert(insertParam); + Response res_drop = client.dropIndex(collectionName); + assert(res_drop.ok()); + DescribeIndexResponse res = client.describeIndex(collectionName); + assert(res.getResponse().ok()); + Index index1 = res.getIndex().get(); + Assert.assertEquals(index1.getIndexType(), defaultIndexType); + } } diff --git a/tests/milvus-java-test/src/main/java/com/TestMix.java b/tests/milvus-java-test/src/main/java/com/TestMix.java index 35f514d136..8487299911 100644 --- a/tests/milvus-java-test/src/main/java/com/TestMix.java +++ b/tests/milvus-java-test/src/main/java/com/TestMix.java @@ -15,53 +15,36 @@ import java.util.stream.Collectors; public class TestMix { private int dimension = 128; private int nb = 100000; + int nq = 10; int n_list = 8192; int n_probe = 20; int top_k = 10; double epsilon = 0.001; int index_file_size = 20; + String indexParam = "{\"nlist\":\"1024\"}"; - public List normalize(List w2v){ - float squareSum = w2v.stream().map(x -> x * x).reduce((float) 0, Float::sum); - final float norm = (float) Math.sqrt(squareSum); - w2v = w2v.stream().map(x -> x / norm).collect(Collectors.toList()); - return w2v; - } + List> vectors = Utils.genVectors(nb, dimension, true); + List> queryVectors = vectors.subList(0,nq); - public List> gen_vectors(int nb, boolean norm) { - List> xb = new ArrayList<>(); - Random random = new Random(); - for (int i = 0; i < nb; ++i) { - List vector = new ArrayList<>(); - for (int j = 0; j < dimension; j++) { - vector.add(random.nextFloat()); - } - if (norm == true) { - vector = normalize(vector); - } - xb.add(vector); - } - return xb; - } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_vectors_threads(MilvusClient client, String tableName) throws InterruptedException { + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_vectors_threads(MilvusClient client, String collectionName) throws InterruptedException { int thread_num = 10; int nq = 5; - List> vectors = gen_vectors(nb, false); - List> queryVectors = vectors.subList(0,nq); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); client.insert(insertParam); - Index index = new Index.Builder().withIndexType(IndexType.IVF_SQ8) - .withNList(n_list) + Index index = new Index.Builder(collectionName, IndexType.IVF_SQ8) + .withParamsInJson(indexParam) .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - client.createIndex(createIndexParam); + client.createIndex(index); ForkJoinPool executor = new ForkJoinPool(); for (int i = 0; i < thread_num; i++) { executor.execute( () -> { - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build(); + String params = "{\"nprobe\":\"1\"}"; + SearchParam searchParam = new SearchParam.Builder(collectionName) + .withFloatVectors(queryVectors) + .withParamsInJson(params) + .withTopK(top_k).build(); SearchResponse res_search = client.search(searchParam); assert (res_search.getResponse().ok()); }); @@ -71,7 +54,7 @@ public class TestMix { } @Test(dataProvider = "DefaultConnectArgs", dataProviderClass = MainClass.class) - public void test_connect_threads(String host, String port) throws ConnectFailedException { + public void test_connect_threads(String host, int port) throws ConnectFailedException { int thread_num = 100; ForkJoinPool executor = new ForkJoinPool(); for (int i = 0; i < thread_num; i++) { @@ -99,11 +82,10 @@ public class TestMix { executor.shutdown(); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_add_vectors_threads(MilvusClient client, String tableName) throws InterruptedException { + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_add_vectors_threads(MilvusClient client, String collectionName) throws InterruptedException { int thread_num = 10; - List> vectors = gen_vectors(nb,false); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); ForkJoinPool executor = new ForkJoinPool(); for (int i = 0; i < thread_num; i++) { executor.execute( @@ -116,19 +98,16 @@ public class TestMix { executor.shutdown(); Thread.sleep(2000); - GetTableRowCountResponse getTableRowCountResponse = client.getTableRowCount(tableName); - Assert.assertEquals(getTableRowCountResponse.getTableRowCount(), thread_num * nb); + GetCollectionRowCountResponse getCollectionRowCountResponse = client.getCollectionRowCount(collectionName); + Assert.assertEquals(getCollectionRowCountResponse.getCollectionRowCount(), thread_num * nb); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_add_vectors_partition_threads(MilvusClient client, String tableName) throws InterruptedException { + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_add_vectors_partition_threads(MilvusClient client, String collectionName) throws InterruptedException { int thread_num = 10; String tag = RandomStringUtils.randomAlphabetic(10); - String partitionName = RandomStringUtils.randomAlphabetic(10); - io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build(); - client.createPartition(partition); - List> vectors = gen_vectors(nb,false); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(tag).build(); + client.createPartition(collectionName, tag); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).withPartitionTag(tag).build(); ForkJoinPool executor = new ForkJoinPool(); for (int i = 0; i < thread_num; i++) { executor.execute( @@ -141,42 +120,39 @@ public class TestMix { executor.shutdown(); Thread.sleep(2000); - GetTableRowCountResponse getTableRowCountResponse = client.getTableRowCount(tableName); - Assert.assertEquals(getTableRowCountResponse.getTableRowCount(), thread_num * nb); + GetCollectionRowCountResponse getCollectionRowCountResponse = client.getCollectionRowCount(collectionName); + Assert.assertEquals(getCollectionRowCountResponse.getCollectionRowCount(), thread_num * nb); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_add_index_vectors_threads(MilvusClient client, String tableName) throws InterruptedException { + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_add_index_vectors_threads(MilvusClient client, String collectionName) throws InterruptedException { int thread_num = 50; - List> vectors = gen_vectors(nb,false); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); ForkJoinPool executor = new ForkJoinPool(); for (int i = 0; i < thread_num; i++) { executor.execute( () -> { InsertResponse res_insert = client.insert(insertParam); - Index index = new Index.Builder().withIndexType(IndexType.IVF_SQ8) - .withNList(n_list) + Index index = new Index.Builder(collectionName, IndexType.IVF_SQ8) + .withParamsInJson("{\"nlist\":\"1024\"}") .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - client.createIndex(createIndexParam); + client.createIndex(index); assert (res_insert.getResponse().ok()); }); } executor.awaitQuiescence(300, TimeUnit.SECONDS); executor.shutdown(); Thread.sleep(2000); - GetTableRowCountResponse getTableRowCountResponse = client.getTableRowCount(tableName); - Assert.assertEquals(getTableRowCountResponse.getTableRowCount(), thread_num * nb); + GetCollectionRowCountResponse getCollectionRowCountResponse = client.getCollectionRowCount(collectionName); + Assert.assertEquals(getCollectionRowCountResponse.getCollectionRowCount(), thread_num * nb); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_add_search_vectors_threads(MilvusClient client, String tableName) throws InterruptedException { + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_add_search_vectors_threads(MilvusClient client, String collectionName) throws InterruptedException { int thread_num = 50; int nq = 5; - List> vectors = gen_vectors(nb, true); List> queryVectors = vectors.subList(0,nq); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); ForkJoinPool executor = new ForkJoinPool(); for (int i = 0; i < thread_num; i++) { executor.execute( @@ -188,14 +164,17 @@ public class TestMix { } catch (InterruptedException e) { e.printStackTrace(); } - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build(); + SearchParam searchParam = new SearchParam.Builder(collectionName) + .withFloatVectors(queryVectors) + .withParamsInJson("{\"nlist\":\"1024\"}") + .withTopK(top_k).build(); SearchResponse res_search = client.search(searchParam); assert (res_search.getResponse().ok()); List> res = client.search(searchParam).getQueryResultsList(); double distance = res.get(0).get(0).getDistance(); - if (tableName.startsWith("L2")) { + if (collectionName.startsWith("L2")) { Assert.assertEquals(distance, 0.0, epsilon); - }else if (tableName.startsWith("IP")) { + }else if (collectionName.startsWith("IP")) { Assert.assertEquals(distance, 1.0, epsilon); } }); @@ -203,14 +182,13 @@ public class TestMix { executor.awaitQuiescence(300, TimeUnit.SECONDS); executor.shutdown(); Thread.sleep(2000); - GetTableRowCountResponse getTableRowCountResponse = client.getTableRowCount(tableName); - Assert.assertEquals(getTableRowCountResponse.getTableRowCount(), thread_num * nb); + GetCollectionRowCountResponse getCollectionRowCountResponse = client.getCollectionRowCount(collectionName); + Assert.assertEquals(getCollectionRowCountResponse.getCollectionRowCount(), thread_num * nb); } @Test(dataProvider = "DefaultConnectArgs", dataProviderClass = MainClass.class) - public void test_create_insert_delete_threads(String host, String port) { + public void test_create_insert_delete_threads(String host, int port) { int thread_num = 100; - List> vectors = gen_vectors(nb,false); ForkJoinPool executor = new ForkJoinPool(); for (int i = 0; i < thread_num; i++) { executor.execute( @@ -226,15 +204,15 @@ public class TestMix { e.printStackTrace(); } assert(client.isConnected()); - String tableName = RandomStringUtils.randomAlphabetic(10); - TableSchema tableSchema = new TableSchema.Builder(tableName, dimension) + String collectionName = RandomStringUtils.randomAlphabetic(10); + CollectionMapping tableSchema = new CollectionMapping.Builder(collectionName, dimension) .withIndexFileSize(index_file_size) .withMetricType(MetricType.IP) .build(); - client.createTable(tableSchema); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + client.createCollection(tableSchema); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); client.insert(insertParam); - Response response = client.dropTable(tableName); + Response response = client.dropCollection(collectionName); Assert.assertTrue(response.ok()); try { client.disconnect(); diff --git a/tests/milvus-java-test/src/main/java/com/TestPS.java b/tests/milvus-java-test/src/main/java/com/TestPS.java index cfce542d1b..527b29599f 100644 --- a/tests/milvus-java-test/src/main/java/com/TestPS.java +++ b/tests/milvus-java-test/src/main/java/com/TestPS.java @@ -3,9 +3,7 @@ package com; import io.milvus.client.*; import org.apache.commons.cli.*; -import java.util.ArrayList; import java.util.List; -import java.util.Random; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -13,7 +11,7 @@ import java.util.stream.Stream; public class TestPS { private static int dimension = 128; - private static String host = "192.168.1.101"; + private static String host = "127.0.0.1"; private static String port = "19530"; public static void setHost(String host) { @@ -24,37 +22,20 @@ public class TestPS { TestPS.port = port; } - public static List normalize(List w2v){ - float squareSum = w2v.stream().map(x -> x * x).reduce((float) 0, Float::sum); - final float norm = (float) Math.sqrt(squareSum); - w2v = w2v.stream().map(x -> x / norm).collect(Collectors.toList()); - return w2v; - } - public static List> gen_vectors(int nb, boolean norm) { - List> xb = new ArrayList<>(); - Random random = new Random(); - for (int i = 0; i < nb; ++i) { - List vector = new ArrayList<>(); - for (int j = 0; j < dimension; j++) { - vector.add(random.nextFloat()); - } - if (norm == true) { - vector = normalize(vector); - } - xb.add(vector); - } - return xb; - } public static void main(String[] args) throws ConnectFailedException { + int nb = 10000; int nq = 5; - int nb = 1; int nprobe = 32; int top_k = 10; int loops = 100000; // int index_file_size = 1024; - String tableName = "sift_1b_2048_128_l2"; + String collectionName = "sift_1b_2048_128_l2"; + + + List> vectors = Utils.genVectors(nb, dimension, true); + List> queryVectors = vectors.subList(0, nq); CommandLineParser parser = new DefaultParser(); Options options = new Options(); @@ -76,16 +57,14 @@ public class TestPS { System.err.println("Parsing failed. Reason: " + exp.getMessage() ); } - List> vectors = gen_vectors(nb, true); - List> queryVectors = gen_vectors(nq, true); MilvusClient client = new MilvusGrpcClient(); ConnectParam connectParam = new ConnectParam.Builder() .withHost(host) - .withPort(port) + .withPort(Integer.parseInt(port)) .build(); client.connect(connectParam); -// String tableName = RandomStringUtils.randomAlphabetic(10); -// TableSchema tableSchema = new TableSchema.Builder(tableName, dimension) +// String collectionName = RandomStringUtils.randomAlphabetic(10); +// TableSchema tableSchema = new TableSchema.Builder(collectionName, dimension) // .withIndexFileSize(index_file_size) // .withMetricType(MetricType.IP) // .build(); @@ -94,7 +73,7 @@ public class TestPS { vectorIds = Stream.iterate(0L, n -> n) .limit(nb) .collect(Collectors.toList()); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withVectorIds(vectorIds).build(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).withVectorIds(vectorIds).build(); ForkJoinPool executor_search = new ForkJoinPool(); for (int i = 0; i < loops; i++) { executor_search.execute( @@ -102,14 +81,14 @@ public class TestPS { InsertResponse res_insert = client.insert(insertParam); assert (res_insert.getResponse().ok()); System.out.println("In insert"); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(nprobe).withTopK(top_k).build(); + SearchParam searchParam = new SearchParam.Builder(collectionName).withFloatVectors(queryVectors).withTopK(top_k).build(); SearchResponse res_search = client.search(searchParam); assert (res_search.getResponse().ok()); }); } executor_search.awaitQuiescence(300, TimeUnit.SECONDS); executor_search.shutdown(); - GetTableRowCountResponse getTableRowCountResponse = client.getTableRowCount(tableName); - System.out.println(getTableRowCountResponse.getTableRowCount()); + GetCollectionRowCountResponse getTableRowCountResponse = client.getCollectionRowCount(collectionName); + System.out.println(getTableRowCountResponse.getCollectionRowCount()); } } diff --git a/tests/milvus-java-test/src/main/java/com/TestPartition.java b/tests/milvus-java-test/src/main/java/com/TestPartition.java new file mode 100644 index 0000000000..eedbd8e04a --- /dev/null +++ b/tests/milvus-java-test/src/main/java/com/TestPartition.java @@ -0,0 +1,156 @@ +package com; + +import io.milvus.client.HasPartitionResponse; +import io.milvus.client.MilvusClient; +import io.milvus.client.Response; +import io.milvus.client.ShowPartitionsResponse; +import org.apache.commons.lang3.RandomStringUtils; +import org.testng.Assert; +import org.testng.annotations.Test; +import java.util.List; + +public class TestPartition { + int dimension = 128; + + // ----------------------------- create partition cases in --------------------------------- + + // create partition + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_create_partition(MilvusClient client, String collectionName) { + String tag = RandomStringUtils.randomAlphabetic(10); + Response createpResponse = client.createPartition(collectionName, tag); + assert (createpResponse.ok()); + // show partitions + List partitions = client.showPartitions(collectionName).getPartitionList(); + System.out.println(partitions); + Assert.assertTrue(partitions.contains(tag)); + } + + // create partition, tag name existed + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_create_partition_tag_name_existed(MilvusClient client, String collectionName) { + String tag = RandomStringUtils.randomAlphabetic(10); + Response createpResponse = client.createPartition(collectionName, tag); + assert (createpResponse.ok()); + Response createpResponseNew = client.createPartition(collectionName, tag); + assert (!createpResponseNew.ok()); + } + + // ----------------------------- has partition cases in --------------------------------- + // has partition, tag name not existed + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_has_partition_tag_name_not_existed(MilvusClient client, String collectionName) { + String tag = RandomStringUtils.randomAlphabetic(10); + Response createpResponse = client.createPartition(collectionName, tag); + assert (createpResponse.ok()); + String tagNew = RandomStringUtils.randomAlphabetic(10); + HasPartitionResponse haspResponse = client.hasPartition(collectionName, tagNew); + assert (haspResponse.ok()); + Assert.assertFalse(haspResponse.hasPartition()); + } + + // has partition, tag name existed + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_has_partition_tag_name_existed(MilvusClient client, String collectionName) { + String tag = RandomStringUtils.randomAlphabetic(10); + Response createpResponse = client.createPartition(collectionName, tag); + assert (createpResponse.ok()); + HasPartitionResponse haspResponse = client.hasPartition(collectionName, tag); + assert (haspResponse.ok()); + Assert.assertTrue(haspResponse.hasPartition()); + } + + // ----------------------------- drop partition cases in --------------------------------- + + // drop a partition created before, drop by partition name + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_drop_partition(MilvusClient client, String collectionName) { + String tag = RandomStringUtils.randomAlphabetic(10); + Response createpResponseNew = client.createPartition(collectionName, tag); + assert (createpResponseNew.ok()); + Response response = client.dropPartition(collectionName, tag); + assert (response.ok()); + // show partitions + System.out.println(client.showPartitions(collectionName).getPartitionList()); + int length = client.showPartitions(collectionName).getPartitionList().size(); + // _default + Assert.assertEquals(length, 1); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_drop_partition_default(MilvusClient client, String collectionName) { + String tag = "_default"; + Response createpResponseNew = client.createPartition(collectionName, tag); + assert (!createpResponseNew.ok()); +// show partitions +// System.out.println(client.showPartitions(collectionName).getPartitionList()); +// int length = client.showPartitions(collectionName).getPartitionList().size(); +// // _default +// Assert.assertEquals(length, 1); + } + + // drop a partition repeat created before, drop by partition name + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_drop_partition_repeat(MilvusClient client, String collectionName) throws InterruptedException { + String tag = RandomStringUtils.randomAlphabetic(10); + Response createpResponse = client.createPartition(collectionName, tag); + assert (createpResponse.ok()); + Response response = client.dropPartition(collectionName, tag); + assert (response.ok()); + Thread.currentThread().sleep(2000); + Response newResponse = client.dropPartition(collectionName, tag); + assert (!newResponse.ok()); + } + + // drop a partition not created before + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_drop_partition_not_existed(MilvusClient client, String collectionName) { + String tag = RandomStringUtils.randomAlphabetic(10); + Response createpResponse = client.createPartition(collectionName, tag); + assert (createpResponse.ok()); + String tagNew = RandomStringUtils.randomAlphabetic(10); + Response response = client.dropPartition(collectionName, tagNew); + assert(!response.ok()); + } + + // drop a partition not created before + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_drop_partition_tag_not_existed(MilvusClient client, String collectionName) { + String tag = RandomStringUtils.randomAlphabetic(10); + Response createpResponse = client.createPartition(collectionName, tag); + assert(createpResponse.ok()); + String newTag = RandomStringUtils.randomAlphabetic(10); + Response response = client.dropPartition(collectionName, newTag); + assert(!response.ok()); + } + + // ----------------------------- show partitions cases in --------------------------------- + + // create partition, then show partitions + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_show_partitions(MilvusClient client, String collectionName) { + String tag = RandomStringUtils.randomAlphabetic(10); + Response createpResponse = client.createPartition(collectionName, tag); + assert (createpResponse.ok()); + ShowPartitionsResponse response = client.showPartitions(collectionName); + assert (response.getResponse().ok()); + Assert.assertTrue(response.getPartitionList().contains(tag)); + } + + // create multi partition, then show partitions + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_show_partitions_multi(MilvusClient client, String collectionName) { + String tag = RandomStringUtils.randomAlphabetic(10); + Response createpResponse = client.createPartition(collectionName, tag); + assert (createpResponse.ok()); + String tagNew = RandomStringUtils.randomAlphabetic(10); + Response newCreatepResponse = client.createPartition(collectionName, tagNew); + assert (newCreatepResponse.ok()); + ShowPartitionsResponse response = client.showPartitions(collectionName); + assert (response.getResponse().ok()); + System.out.println(response.getPartitionList()); + Assert.assertTrue(response.getPartitionList().contains(tag)); + Assert.assertTrue(response.getPartitionList().contains(tagNew)); + } + +} diff --git a/tests/milvus-java-test/src/main/java/com/TestPing.java b/tests/milvus-java-test/src/main/java/com/TestPing.java index 1ed462e47d..de5a75f419 100644 --- a/tests/milvus-java-test/src/main/java/com/TestPing.java +++ b/tests/milvus-java-test/src/main/java/com/TestPing.java @@ -5,7 +5,7 @@ import org.testng.annotations.Test; public class TestPing { @Test(dataProvider = "DefaultConnectArgs", dataProviderClass = MainClass.class) - public void test_server_status(String host, String port) throws ConnectFailedException { + public void test_server_status(String host, int port) throws ConnectFailedException { System.out.println("Host: "+host+", Port: "+port); MilvusClient client = new MilvusGrpcClient(); ConnectParam connectParam = new ConnectParam.Builder() @@ -18,7 +18,7 @@ public class TestPing { } @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class) - public void test_server_status_without_connected(MilvusGrpcClient client, String tableName){ + public void test_server_status_without_connected(MilvusClient client, String collectionName) throws ConnectFailedException { Response res = client.getServerStatus(); assert (!res.ok()); } diff --git a/tests/milvus-java-test/src/main/java/com/TestSearchByIds.java b/tests/milvus-java-test/src/main/java/com/TestSearchByIds.java new file mode 100644 index 0000000000..51f0a0b13f --- /dev/null +++ b/tests/milvus-java-test/src/main/java/com/TestSearchByIds.java @@ -0,0 +1,255 @@ +package com; + +import io.milvus.client.*; +import org.apache.commons.lang3.RandomStringUtils; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class TestSearchByIds { + int dimension = 128; + int n_list = 1024; + int default_n_list = 16384; + int nb = 10000; + int n_probe = 20; + int top_k = 10; + int nq = 5; + double epsilon = 0.001; + IndexType indexType = IndexType.IVF_SQ8; + IndexType defaultIndexType = IndexType.FLAT; + List default_ids = Utils.toListIds(1111); + List> vectors = Utils.genVectors(nb, dimension, true); + List vectorsBinary = Utils.genBinaryVectors(nb, dimension); + String indexParam = Utils.setIndexParam(n_list); + public String searchParamStr = Utils.setSearchParam(n_probe); + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_collection_not_existed(MilvusClient client, String collectionName) { + String collectionNameNew = collectionName + "_"; + SearchByIdsParam searchParam = new SearchByIdsParam.Builder(collectionNameNew) + .withParamsInJson(searchParamStr) + .withTopK(top_k) + .withIDs(default_ids) + .build(); + SearchResponse res_search = client.searchByIds(searchParam); + assert (!res_search.getResponse().ok()); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_collection_empty(MilvusClient client, String collectionName) { + SearchByIdsParam searchParam = new SearchByIdsParam.Builder(collectionName) + .withParamsInJson(searchParamStr) + .withTopK(top_k) + .withIDs(default_ids) + .build(); + SearchResponse res_search = client.searchByIds(searchParam); + assert (!res_search.getResponse().ok()); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_no_result(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + client.insert(insertParam); + client.flush(collectionName); + SearchByIdsParam searchParam = new SearchByIdsParam.Builder(collectionName) + .withParamsInJson(searchParamStr) + .withTopK(top_k) + .withIDs(default_ids) + .build(); + List> res_search = client.searchByIds(searchParam).getQueryResultsList(); + assert (client.searchByIds(searchParam).getResponse().ok()); + Assert.assertEquals(res_search.get(0).size(), top_k); + Assert.assertEquals(res_search.size(), default_ids.size()); + Assert.assertEquals(res_search.get(0).get(0).getVectorId(), -1); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_index_IVFLAT(MilvusClient client, String collectionName) { + IndexType indexType = IndexType.IVFLAT; + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + InsertResponse res_insert = client.insert(insertParam); + client.flush(collectionName); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + client.createIndex(index); + SearchByIdsParam searchParam = new SearchByIdsParam.Builder(collectionName) + .withParamsInJson(searchParamStr) + .withTopK(top_k) + .withIDs(res_insert.getVectorIds()) + .build(); + List> res_search = client.searchByIds(searchParam).getQueryResultsList(); + for (int i=0; i queryTags = new ArrayList<>(); + queryTags.add(tag); + SearchByIdsParam searchParam = new SearchByIdsParam.Builder(collectionName) + .withParamsInJson(searchParamStr) + .withTopK(top_k) + .withIDs(Utils.toListIds(res_insert.getVectorIds().get(0))) + .withPartitionTags(queryTags) + .build(); + List> res_search = client.searchByIds(searchParam).getQueryResultsList(); + double distance = res_search.get(0).get(0).getDistance(); + if (collectionName.startsWith("L2")) { + Assert.assertEquals(distance, 0.0, epsilon); + }else if (collectionName.startsWith("IP")) { + Assert.assertEquals(distance, 1.0, epsilon); + } + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_partition_not_exited(MilvusClient client, String collectionName) { + String tag = RandomStringUtils.randomAlphabetic(10); + client.createPartition(collectionName, tag); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).withPartitionTag(tag).build(); + client.insert(insertParam); + client.flush(collectionName); + String tagNew = RandomStringUtils.randomAlphabetic(10); + List queryTags = new ArrayList<>(); + queryTags.add(tagNew); + SearchByIdsParam searchParam = new SearchByIdsParam.Builder(collectionName) + .withParamsInJson(searchParamStr) + .withTopK(top_k) + .withIDs(default_ids) + .withPartitionTags(queryTags) + .build(); + SearchResponse res_search = client.searchByIds(searchParam); + assert (!res_search.getResponse().ok()); + Assert.assertEquals(res_search.getQueryResultsList().size(), 0); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_partition_empty(MilvusClient client, String collectionName) { + String tag = RandomStringUtils.randomAlphabetic(10); + client.createPartition(collectionName, tag); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).withPartitionTag(tag).build(); + client.insert(insertParam); + client.flush(collectionName); + List queryTags = new ArrayList<>(); + queryTags.add(tag); + SearchByIdsParam searchParam = new SearchByIdsParam.Builder(collectionName) + .withParamsInJson(searchParamStr) + .withTopK(top_k) + .withIDs(default_ids) + .withPartitionTags(queryTags) + .build(); + SearchResponse res_search = client.searchByIds(searchParam); + assert (res_search.getResponse().ok()); + Assert.assertEquals(res_search.getQueryResultsList().size(), 1); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_invalid_n_probe(MilvusClient client, String collectionName) { + int n_probe_new = 0; + String searchParamStrNew = Utils.setSearchParam(n_probe_new); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + client.insert(insertParam); + SearchByIdsParam searchParam = new SearchByIdsParam.Builder(collectionName) + .withParamsInJson(searchParamStrNew) + .withTopK(top_k) + .withIDs(default_ids) + .build(); + SearchResponse res_search = client.searchByIds(searchParam); + assert (!res_search.getResponse().ok()); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_invalid_top_k(MilvusClient client, String collectionName) { + int top_k_new = 0; + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + client.insert(insertParam); + SearchByIdsParam searchParam = new SearchByIdsParam.Builder(collectionName) + .withParamsInJson(searchParamStr) + .withTopK(top_k_new) + .withIDs(default_ids) + .build(); + SearchResponse res_search = client.searchByIds(searchParam); + assert (!res_search.getResponse().ok()); + } + + // Binary tests + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_search_collection_not_existed_binary(MilvusClient client, String collectionName) { + String collectionNameNew = collectionName + "_"; + SearchByIdsParam searchParam = new SearchByIdsParam.Builder(collectionNameNew) + .withParamsInJson(searchParamStr) + .withTopK(top_k) + .withIDs(default_ids) + .build(); + SearchResponse res_search = client.searchByIds(searchParam); + assert (!res_search.getResponse().ok()); + } + + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_search_ids_binary(MilvusClient client, String collectionName) { + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + InsertResponse res_insert = client.insert(insertParam); + client.flush(collectionName); + SearchByIdsParam searchParam = new SearchByIdsParam.Builder(collectionName) + .withParamsInJson(searchParamStr) + .withTopK(top_k) + .withIDs(res_insert.getVectorIds()) + .build(); + List> res_search = client.searchByIds(searchParam).getQueryResultsList(); + for (int i = 0; i < top_k; i++) { + long insert_id = res_insert.getVectorIds().get(i); + long get_id = res_search.get(i).get(0).getVectorId(); + System.out.println(get_id); + Assert.assertEquals(insert_id, get_id); + } + } + + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_search_invalid_n_probe_binary(MilvusClient client, String collectionName) { + int n_probe_new = 0; + String searchParamStrNew = Utils.setSearchParam(n_probe_new); + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + client.insert(insertParam); + SearchByIdsParam searchParam = new SearchByIdsParam.Builder(collectionName) + .withParamsInJson(searchParamStrNew) + .withTopK(top_k) + .withIDs(default_ids) + .build(); + SearchResponse res_search = client.searchByIds(searchParam); + assert (!res_search.getResponse().ok()); + } + + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_search_invalid_top_k_binary(MilvusClient client, String collectionName) { + int top_k_new = 0; + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + client.insert(insertParam); + SearchByIdsParam searchParam = new SearchByIdsParam.Builder(collectionName) + .withParamsInJson(searchParamStr) + .withTopK(top_k_new) + .withIDs(default_ids) + .build(); + SearchResponse res_search = client.searchByIds(searchParam); + assert (!res_search.getResponse().ok()); + } + +} diff --git a/tests/milvus-java-test/src/main/java/com/TestSearchVectors.java b/tests/milvus-java-test/src/main/java/com/TestSearchVectors.java index 3cd4f7de93..750296c094 100644 --- a/tests/milvus-java-test/src/main/java/com/TestSearchVectors.java +++ b/tests/milvus-java-test/src/main/java/com/TestSearchVectors.java @@ -5,579 +5,295 @@ import org.apache.commons.lang3.RandomStringUtils; import org.testng.Assert; import org.testng.annotations.Test; +import java.nio.ByteBuffer; import java.util.*; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; public class TestSearchVectors { - int index_file_size = 10; int dimension = 128; int n_list = 1024; int default_n_list = 16384; - int nb = 100000; + int nb = 10000; int n_probe = 20; int top_k = 10; + int nq = 5; double epsilon = 0.001; IndexType indexType = IndexType.IVF_SQ8; IndexType defaultIndexType = IndexType.FLAT; + List> vectors = Utils.genVectors(nb, dimension, true); + List vectorsBinary = Utils.genBinaryVectors(nb, dimension); + List> queryVectors = vectors.subList(0, nq); + List queryVectorsBinary = vectorsBinary.subList(0, nq); + String indexParam = Utils.setIndexParam(n_list); + public String searchParamStr = Utils.setSearchParam(n_probe); - - public List normalize(List w2v){ - float squareSum = w2v.stream().map(x -> x * x).reduce((float) 0, Float::sum); - final float norm = (float) Math.sqrt(squareSum); - w2v = w2v.stream().map(x -> x / norm).collect(Collectors.toList()); - return w2v; - } - - public List> gen_vectors(int nb, boolean norm) { - List> xb = new ArrayList<>(); - Random random = new Random(); - for (int i = 0; i < nb; ++i) { - List vector = new ArrayList<>(); - for (int j = 0; j < dimension; j++) { - vector.add(random.nextFloat()); - } - if (norm == true) { - vector = normalize(vector); - } - xb.add(vector); - } - return xb; - } - - public static Date getDeltaDate(int delta) { - Date today = new Date(); - Calendar c = Calendar.getInstance(); - c.setTime(today); - c.add(Calendar.DAY_OF_MONTH, delta); - return c.getTime(); - } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_table_not_existed(MilvusClient client, String tableName) { - String tableNameNew = tableName + "_"; - int nq = 5; - int nb = 100; - List> vectors = gen_vectors(nb, false); - List> queryVectors = vectors.subList(0,nq); - SearchParam searchParam = new SearchParam.Builder(tableNameNew, queryVectors).withNProbe(n_probe).withTopK(top_k).build(); + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_collection_not_existed(MilvusClient client, String collectionName) { + String collectionNameNew = collectionName + "_"; + SearchParam searchParam = new SearchParam.Builder(collectionNameNew) + .withFloatVectors(queryVectors) + .withParamsInJson(searchParamStr) + .withTopK(top_k).build(); SearchResponse res_search = client.search(searchParam); assert (!res_search.getResponse().ok()); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_index_IVFLAT(MilvusClient client, String tableName) { + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_index_IVFLAT(MilvusClient client, String collectionName) { IndexType indexType = IndexType.IVFLAT; - int nq = 5; - List> vectors = gen_vectors(nb, false); - List> queryVectors = vectors.subList(0,nq); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - client.createIndex(createIndexParam); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build(); + client.flush(collectionName); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + client.createIndex(index); + SearchParam searchParam = new SearchParam.Builder(collectionName) + .withFloatVectors(queryVectors) + .withParamsInJson(searchParamStr) + .withTopK(top_k).build(); List> res_search = client.search(searchParam).getQueryResultsList(); Assert.assertEquals(res_search.size(), nq); Assert.assertEquals(res_search.get(0).size(), top_k); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_ids_IVFLAT(MilvusClient client, String tableName) { + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_ids_IVFLAT(MilvusClient client, String collectionName) { IndexType indexType = IndexType.IVFLAT; - int nq = 5; - List> vectors = gen_vectors(nb, true); - List> queryVectors = vectors.subList(0,nq); List vectorIds; vectorIds = Stream.iterate(0L, n -> n) .limit(nb) .collect(Collectors.toList()); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withVectorIds(vectorIds).build(); - client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - client.createIndex(createIndexParam); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).withVectorIds(vectorIds).build(); + InsertResponse res = client.insert(insertParam); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + client.createIndex(index); + SearchParam searchParam = new SearchParam.Builder(collectionName) + .withFloatVectors(queryVectors) + .withParamsInJson(searchParamStr) + .withTopK(top_k).build(); List> res_search = client.search(searchParam).getQueryResultsList(); Assert.assertEquals(res_search.get(0).get(0).getVectorId(), 0L); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_IVFLAT(MilvusClient client, String tableName) throws InterruptedException { + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_distance_IVFLAT(MilvusClient client, String collectionName) { IndexType indexType = IndexType.IVFLAT; - int nq = 5; - List> vectors = gen_vectors(nb, false); - List> queryVectors = vectors.subList(0,nq); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); - client.insert(insertParam); - Thread.sleep(2000); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build(); - List> res_search = client.search(searchParam).getQueryResultsList(); - Assert.assertEquals(res_search.size(), nq); - Assert.assertEquals(res_search.get(0).size(), top_k); - } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_distance_IVFLAT(MilvusClient client, String tableName) { - IndexType indexType = IndexType.IVFLAT; - int nq = 5; - List> vectors = gen_vectors(nb, true); - List> queryVectors = vectors.subList(0,nq); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); - client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - client.createIndex(createIndexParam); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + InsertResponse res = client.insert(insertParam); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + client.createIndex(index); + SearchParam searchParam = new SearchParam.Builder(collectionName) + .withFloatVectors(queryVectors) + .withParamsInJson(searchParamStr) + .withTopK(top_k).build(); List> res_search = client.search(searchParam).getQueryResultsList(); double distance = res_search.get(0).get(0).getDistance(); - if (tableName.startsWith("L2")) { + if (collectionName.startsWith("L2")) { Assert.assertEquals(distance, 0.0, epsilon); - }else if (tableName.startsWith("IP")) { + }else if (collectionName.startsWith("IP")) { Assert.assertEquals(distance, 1.0, epsilon); } } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_distance_partition(MilvusClient client, String tableName) { + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_partition(MilvusClient client, String collectionName) { IndexType indexType = IndexType.IVFLAT; - int nq = 5; String tag = RandomStringUtils.randomAlphabetic(10); - List> vectors = gen_vectors(nb, true); - List> queryVectors = vectors.subList(0,nq); - String partitionName = RandomStringUtils.randomAlphabetic(10); - io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build(); - client.createPartition(partition); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(tag).build(); - client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - client.createIndex(createIndexParam); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build(); + Response createpResponse = client.createPartition(collectionName, tag); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + InsertResponse res = client.insert(insertParam); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + client.createIndex(index); + SearchParam searchParam = new SearchParam.Builder(collectionName) + .withFloatVectors(queryVectors) + .withParamsInJson(searchParamStr) + .withTopK(top_k).build(); List> res_search = client.search(searchParam).getQueryResultsList(); double distance = res_search.get(0).get(0).getDistance(); - if (tableName.startsWith("L2")) { + if (collectionName.startsWith("L2")) { Assert.assertEquals(distance, 0.0, epsilon); - }else if (tableName.startsWith("IP")) { + }else if (collectionName.startsWith("IP")) { Assert.assertEquals(distance, 1.0, epsilon); } } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_distance_partition_not_exited(MilvusClient client, String tableName) { + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_partition_not_exited(MilvusClient client, String collectionName) { IndexType indexType = IndexType.IVFLAT; - int nq = 5; String tag = RandomStringUtils.randomAlphabetic(10); - List> vectors = gen_vectors(nb, true); - List> queryVectors = vectors.subList(0,nq); - String partitionName = RandomStringUtils.randomAlphabetic(10); - io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build(); - client.createPartition(partition); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(tag).build(); - client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - client.createIndex(createIndexParam); + Response createpResponse = client.createPartition(collectionName, tag); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); + InsertResponse res = client.insert(insertParam); String tagNew = RandomStringUtils.randomAlphabetic(10); List queryTags = new ArrayList<>(); queryTags.add(tagNew); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withPartitionTags(queryTags).build(); + SearchParam searchParam = new SearchParam.Builder(collectionName) + .withFloatVectors(queryVectors) + .withParamsInJson(searchParamStr) + .withPartitionTags(queryTags) + .withTopK(top_k).build(); SearchResponse res_search = client.search(searchParam); - assert (res_search.getResponse().ok()); + assert (!res_search.getResponse().ok()); Assert.assertEquals(res_search.getQueryResultsList().size(), 0); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_distance_partition_empty(MilvusClient client, String tableName) { + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_partition_empty(MilvusClient client, String collectionName) { IndexType indexType = IndexType.IVFLAT; - int nq = 5; String tag = RandomStringUtils.randomAlphabetic(10); - List> vectors = gen_vectors(nb, true); - List> queryVectors = vectors.subList(0,nq); - String partitionName = RandomStringUtils.randomAlphabetic(10); - io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build(); - client.createPartition(partition); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(tag).build(); - client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - client.createIndex(createIndexParam); + Response createpResponse = client.createPartition(collectionName, tag); String tagNew = ""; List queryTags = new ArrayList<>(); queryTags.add(tagNew); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withPartitionTags(queryTags).build(); + SearchParam searchParam = new SearchParam.Builder(collectionName) + .withFloatVectors(queryVectors) + .withParamsInJson(searchParamStr) + .withPartitionTags(queryTags) + .withTopK(top_k).build(); SearchResponse res_search = client.search(searchParam); assert (!res_search.getResponse().ok()); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_distance_partition_A(MilvusClient client, String tableName) throws InterruptedException { -// IndexType indexType = IndexType.IVFLAT; - String tag = RandomStringUtils.randomAlphabetic(10); - String tagNew = RandomStringUtils.randomAlphabetic(10); - List> vectors = gen_vectors(nb, true); - List> vectorsNew = gen_vectors(nb, true); - String partitionName = RandomStringUtils.randomAlphabetic(10); - String partitionNameNew = RandomStringUtils.randomAlphabetic(10); - io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build(); - io.milvus.client.Partition partitionNew = new io.milvus.client.Partition.Builder(tableName, partitionNameNew, tagNew).build(); - client.createPartition(partition); - client.createPartition(partitionNew); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(tag).build(); + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_distance_FLAT(MilvusClient client, String collectionName) { + IndexType indexType = IndexType.FLAT; + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); InsertResponse res = client.insert(insertParam); - System.out.println(res.getVectorIds().size()); - InsertParam insertParamNew = new InsertParam.Builder(tableName, vectorsNew).withPartitionTag(tagNew).build(); - InsertResponse resNew = client.insert(insertParamNew); - TimeUnit.SECONDS.sleep(2); - System.out.println(resNew.getVectorIds().size()); - List queryTags = new ArrayList<>(); - queryTags.add(tag); - List> queryVectors; - queryVectors = vectors.subList(0,2); - queryVectors.add(vectorsNew.get(0)); - System.out.println(queryVectors.size()); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withPartitionTags(queryTags).build(); - List> res_search = client.search(searchParam).getResultIdsList(); - System.out.println(res_search.get(0)); - System.out.println(res.getVectorIds()); -// System.out.println(res_search.get(2)); - Assert.assertTrue(res.getVectorIds().containsAll(res_search.get(0))); - Assert.assertTrue(resNew.getVectorIds().contains(res_search.get(2))); - } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_index_IVFSQ8(MilvusClient client, String tableName) { - IndexType indexType = IndexType.IVF_SQ8; - int nq = 5; - List> vectors = gen_vectors(nb, false); - List> queryVectors = vectors.subList(0,nq); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); - client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - client.createIndex(createIndexParam); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build(); - List> res_search = client.search(searchParam).getQueryResultsList(); - Assert.assertEquals(res_search.size(), nq); - Assert.assertEquals(res_search.get(0).size(), top_k); - } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_IVFSQ8(MilvusClient client, String tableName) throws InterruptedException { - IndexType indexType = IndexType.IVF_SQ8; - int nq = 5; - List> vectors = gen_vectors(nb, false); - List> queryVectors = vectors.subList(0,nq); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); - client.insert(insertParam); - Thread.sleep(1000); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build(); - List> res_search = client.search(searchParam).getQueryResultsList(); - Assert.assertEquals(res_search.size(), nq); - Assert.assertEquals(res_search.get(0).size(), top_k); - } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_distance_IVFSQ8(MilvusClient client, String tableName) { - IndexType indexType = IndexType.IVF_SQ8; - int nq = 5; - int nb = 1000; - List> vectors = gen_vectors(nb, true); - List> queryVectors = vectors.subList(0,nq); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); - client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(default_n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - client.createIndex(createIndexParam); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build(); - List> res_search = client.search(searchParam).getResultDistancesList(); - for (int i = 0; i < nq; i++) { - double distance = res_search.get(i).get(0); - System.out.println(distance); - if (tableName.startsWith("L2")) { - Assert.assertEquals(distance, 0.0, epsilon); - }else if (tableName.startsWith("IP")) { - Assert.assertEquals(distance, 1.0, epsilon); - } - } - } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_index_FLAT(MilvusClient client, String tableName) { - IndexType indexType = IndexType.FLAT; - int nq = 5; - List> vectors = gen_vectors(nb, false); - List> queryVectors = vectors.subList(0,nq); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); - client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - client.createIndex(createIndexParam); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build(); - List> res_search = client.search(searchParam).getQueryResultsList(); - Assert.assertEquals(res_search.size(), nq); - Assert.assertEquals(res_search.get(0).size(), top_k); - } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_FLAT(MilvusClient client, String tableName) throws InterruptedException { - IndexType indexType = IndexType.FLAT; - int nq = 5; - List> vectors = gen_vectors(nb, false); - List> queryVectors = vectors.subList(0,nq); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); - client.insert(insertParam); - Thread.sleep(1000); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build(); - List> res_search = client.search(searchParam).getQueryResultsList(); - Assert.assertEquals(res_search.size(), nq); - Assert.assertEquals(res_search.get(0).size(), top_k); - } - -// @Test(dataProvider = "Table", dataProviderClass = MainClass.class) -// public void test_search_FLAT_timeout(MilvusClient client, String tableName) throws InterruptedException { -// IndexType indexType = IndexType.FLAT; -// int nb = 100000; -// int nq = 1000; -// int top_k = 2048; -// List> vectors = gen_vectors(nb, false); -// List> vectors = gen_vectors(nb, false); -// List> queryVectors = vectors.subList(0,nq); -// InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); -// client.insert(insertParam); -// Thread.sleep(1000); -// SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withTimeout(1).build(); -// System.out.println(new Date()); -// SearchResponse res_search = client.search(searchParam); -// assert (!res_search.getResponse().ok()); -// } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_FLAT_big_data_size(MilvusClient client, String tableName) throws InterruptedException { - IndexType indexType = IndexType.FLAT; - int nb = 100000; - int nq = 2000; - int top_k = 2048; - List> vectors = gen_vectors(nb, false); - List> queryVectors = vectors.subList(0,nq); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); - client.insert(insertParam); - Thread.sleep(1000); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build(); - System.out.println(new Date()); - SearchResponse res_search = client.search(searchParam); - assert (res_search.getResponse().ok()); - } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_distance_FLAT(MilvusClient client, String tableName) { - IndexType indexType = IndexType.FLAT; - int nq = 5; - List> vectors = gen_vectors(nb, true); - List> queryVectors = vectors.subList(0,nq); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); - client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - client.createIndex(createIndexParam); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build(); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + client.createIndex(index); + SearchParam searchParam = new SearchParam.Builder(collectionName) + .withFloatVectors(queryVectors) + .withParamsInJson(searchParamStr) + .withTopK(top_k).build(); List> res_search = client.search(searchParam).getQueryResultsList(); double distance = res_search.get(0).get(0).getDistance(); - if (tableName.startsWith("L2")) { + if (collectionName.startsWith("L2")) { Assert.assertEquals(distance, 0.0, epsilon); - }else if (tableName.startsWith("IP")) { + }else if (collectionName.startsWith("IP")) { Assert.assertEquals(distance, 1.0, epsilon); } } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_invalid_n_probe(MilvusClient client, String tableName) { - IndexType indexType = IndexType.IVF_SQ8; - int nq = 5; - int n_probe_new = 0; - List> vectors = gen_vectors(nb, false); - List> queryVectors = vectors.subList(0,nq); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_invalid_n_probe(MilvusClient client, String collectionName) { + int n_probe_new = -1; + String searchParamStrNew = Utils.setSearchParam(n_probe_new); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - client.createIndex(createIndexParam); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe_new).withTopK(top_k).build(); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + client.createIndex(index); + SearchParam searchParam = new SearchParam.Builder(collectionName) + .withFloatVectors(queryVectors) + .withParamsInJson(searchParamStrNew) + .withTopK(top_k).build(); SearchResponse res_search = client.search(searchParam); assert (!res_search.getResponse().ok()); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_invalid_top_k(MilvusClient client, String tableName) { - IndexType indexType = IndexType.IVF_SQ8; - int nq = 5; + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_invalid_top_k(MilvusClient client, String collectionName) { int top_k_new = 0; - List> vectors = gen_vectors(nb, false); - List> queryVectors = vectors.subList(0,nq); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).build(); client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - client.createIndex(createIndexParam); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k_new).build(); + SearchParam searchParam = new SearchParam.Builder(collectionName) + .withFloatVectors(queryVectors) + .withParamsInJson(searchParamStr) + .withTopK(top_k_new).build(); SearchResponse res_search = client.search(searchParam); assert (!res_search.getResponse().ok()); } -// @Test(dataProvider = "Table", dataProviderClass = MainClass.class) -// public void test_search_invalid_query_vectors(MilvusClient client, String tableName) throws InterruptedException { -// IndexType indexType = IndexType.IVF_SQ8; -// int nq = 5; -// List> vectors = gen_vectors(nb, false); -// InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); -// client.insert(insertParam); -// Index index = new Index.Builder().withIndexType(indexType) -// .withNList(n_list) -// .build(); -// CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); -// client.createIndex(createIndexParam); -// TableParam tableParam = new TableParam.Builder(tableName).build(); -// SearchParam searchParam = new SearchParam.Builder(tableName, null).withNProbe(n_probe).withTopK(top_k).build(); -// SearchResponse res_search = client.search(searchParam); -// assert (!res_search.getResponse().ok()); -// } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_index_range(MilvusClient client, String tableName) { - IndexType indexType = IndexType.IVF_SQ8; - int nq = 5; - List> vectors = gen_vectors(nb, false); - List> queryVectors = vectors.subList(0,nq); - List dateRange = new ArrayList<>(); - dateRange.add(new DateRange(getDeltaDate(-1), getDeltaDate(1))); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); - client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - client.createIndex(createIndexParam); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withDateRanges(dateRange).build(); - SearchResponse res_search = client.search(searchParam); - assert (res_search.getResponse().ok()); - List> res = client.search(searchParam).getQueryResultsList(); - Assert.assertEquals(res.size(), nq); - Assert.assertEquals(res.get(0).size(), top_k); - } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_range(MilvusClient client, String tableName) throws InterruptedException { - int nq = 5; - List> vectors = gen_vectors(nb, false); - List> queryVectors = vectors.subList(0,nq); - List dateRange = new ArrayList<>(); - dateRange.add(new DateRange(getDeltaDate(-1), getDeltaDate(1))); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); - client.insert(insertParam); - Thread.sleep(1000); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withDateRanges(dateRange).build(); - SearchResponse res_search = client.search(searchParam); - assert (res_search.getResponse().ok()); - List> res = client.search(searchParam).getQueryResultsList(); - Assert.assertEquals(res.size(), nq); - Assert.assertEquals(res.get(0).size(), top_k); - } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_index_range_no_result(MilvusClient client, String tableName) throws InterruptedException { - IndexType indexType = IndexType.IVF_SQ8; - int nq = 5; - List> vectors = gen_vectors(nb, false); - List> queryVectors = vectors.subList(0,nq); - List dateRange = new ArrayList<>(); - dateRange.add(new DateRange(getDeltaDate(-3), getDeltaDate(-1))); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); - client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - client.createIndex(createIndexParam); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withDateRanges(dateRange).build(); - SearchResponse res_search = client.search(searchParam); - assert (res_search.getResponse().ok()); - List> res = client.search(searchParam).getQueryResultsList(); - Assert.assertEquals(res.size(), 0); - } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_range_no_result(MilvusClient client, String tableName) throws InterruptedException { - int nq = 5; - List> vectors = gen_vectors(nb, false); - List> queryVectors = vectors.subList(0,nq); - List dateRange = new ArrayList<>(); - dateRange.add(new DateRange(getDeltaDate(-3), getDeltaDate(-1))); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); - client.insert(insertParam); - Thread.sleep(1000); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withDateRanges(dateRange).build(); - SearchResponse res_search = client.search(searchParam); - assert (res_search.getResponse().ok()); - List> res = client.search(searchParam).getQueryResultsList(); - Assert.assertEquals(res.size(), 0); - } - - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_index_range_invalid(MilvusClient client, String tableName) throws InterruptedException { - IndexType indexType = IndexType.IVF_SQ8; - int nq = 5; - List> vectors = gen_vectors(nb, false); - List> queryVectors = vectors.subList(0,nq); - List dateRange = new ArrayList<>(); - dateRange.add(new DateRange(getDeltaDate(2), getDeltaDate(-1))); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); - client.insert(insertParam); - Index index = new Index.Builder().withIndexType(indexType) - .withNList(n_list) - .build(); - CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); - client.createIndex(createIndexParam); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withDateRanges(dateRange).build(); + // Binary tests + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_search_collection_not_existed_binary(MilvusClient client, String collectionName) { + String collectionNameNew = collectionName + "_"; + SearchParam searchParam = new SearchParam.Builder(collectionNameNew) + .withBinaryVectors(queryVectorsBinary) + .withParamsInJson(searchParamStr) + .withTopK(top_k).build(); SearchResponse res_search = client.search(searchParam); assert (!res_search.getResponse().ok()); } - @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_range_invalid(MilvusClient client, String tableName) throws InterruptedException { - int nq = 5; - List> vectors = gen_vectors(nb, false); - List> queryVectors = vectors.subList(0,nq); - List dateRange = new ArrayList<>(); - dateRange.add(new DateRange(getDeltaDate(2), getDeltaDate(-1))); - InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_search_index_IVFLAT_binary(MilvusClient client, String collectionName) { + IndexType indexType = IndexType.IVFLAT; + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + InsertResponse res = client.insert(insertParam); + client.flush(collectionName); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + client.createIndex(index); + SearchParam searchParam = new SearchParam.Builder(collectionName) + .withBinaryVectors(queryVectorsBinary) + .withParamsInJson(searchParamStr) + .withTopK(top_k).build(); + List> res_search = client.search(searchParam).getQueryResultsList(); + Assert.assertEquals(res_search.size(), nq); + Assert.assertEquals(res_search.get(0).size(), top_k); + } + + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_search_ids_IVFLAT_binary(MilvusClient client, String collectionName) { + IndexType indexType = IndexType.IVFLAT; + List vectorIds; + vectorIds = Stream.iterate(0L, n -> n) + .limit(nb) + .collect(Collectors.toList()); + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).withVectorIds(vectorIds).build(); + InsertResponse res = client.insert(insertParam); + Index index = new Index.Builder(collectionName, indexType).withParamsInJson(indexParam).build(); + client.createIndex(index); + SearchParam searchParam = new SearchParam.Builder(collectionName) + .withBinaryVectors(queryVectorsBinary) + .withParamsInJson(searchParamStr) + .withTopK(top_k).build(); + List> res_search = client.search(searchParam).getQueryResultsList(); + Assert.assertEquals(res_search.get(0).get(0).getVectorId(), 0L); + } + + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_search_partition_not_existed_binary(MilvusClient client, String collectionName) { + IndexType indexType = IndexType.IVFLAT; + String tag = RandomStringUtils.randomAlphabetic(10); + Response createpResponse = client.createPartition(collectionName, tag); + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + InsertResponse res = client.insert(insertParam); + String tagNew = RandomStringUtils.randomAlphabetic(10); + List queryTags = new ArrayList<>(); + queryTags.add(tagNew); + SearchParam searchParam = new SearchParam.Builder(collectionName) + .withBinaryVectors(queryVectorsBinary) + .withParamsInJson(searchParamStr) + .withPartitionTags(queryTags) + .withTopK(top_k).build(); + SearchResponse res_search = client.search(searchParam); + assert (!res_search.getResponse().ok()); + Assert.assertEquals(res_search.getQueryResultsList().size(), 0); + } + + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_search_invalid_n_probe_binary(MilvusClient client, String collectionName) { + int n_probe_new = 0; + String searchParamStrNew = Utils.setSearchParam(n_probe_new); + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); client.insert(insertParam); - Thread.sleep(1000); - SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withDateRanges(dateRange).build(); + SearchParam searchParam = new SearchParam.Builder(collectionName) + .withBinaryVectors(queryVectorsBinary) + .withParamsInJson(searchParamStrNew) + .withTopK(top_k).build(); + SearchResponse res_search = client.search(searchParam); + assert (res_search.getResponse().ok()); + } + + @Test(dataProvider = "BinaryCollection", dataProviderClass = MainClass.class) + public void test_search_invalid_top_k_binary(MilvusClient client, String collectionName) { + int top_k_new = 0; + InsertParam insertParam = new InsertParam.Builder(collectionName).withBinaryVectors(vectorsBinary).build(); + client.insert(insertParam); + SearchParam searchParam = new SearchParam.Builder(collectionName) + .withBinaryVectors(queryVectorsBinary) + .withParamsInJson(searchParamStr) + .withTopK(top_k_new).build(); SearchResponse res_search = client.search(searchParam); assert (!res_search.getResponse().ok()); } diff --git a/tests/milvus-java-test/src/main/java/com/Utils.java b/tests/milvus-java-test/src/main/java/com/Utils.java new file mode 100644 index 0000000000..bb800ad5aa --- /dev/null +++ b/tests/milvus-java-test/src/main/java/com/Utils.java @@ -0,0 +1,79 @@ +package com; + +import com.alibaba.fastjson.JSONObject; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; + +public class Utils { + + public static List normalize(List w2v){ + float squareSum = w2v.stream().map(x -> x * x).reduce((float) 0, Float::sum); + final float norm = (float) Math.sqrt(squareSum); + w2v = w2v.stream().map(x -> x / norm).collect(Collectors.toList()); + return w2v; + } + + public static List> genVectors(int nb, int dimension, boolean norm) { + List> xb = new ArrayList<>(); + Random random = new Random(); + for (int i = 0; i < nb; ++i) { + List vector = new ArrayList<>(); + for (int j = 0; j < dimension; j++) { + vector.add(random.nextFloat()); + } + if (norm == true) { + vector = normalize(vector); + } + xb.add(vector); + } + return xb; + } + + static List genBinaryVectors(long vectorCount, long dimension) { + Random random = new Random(); + List vectors = new ArrayList<>(); + final long dimensionInByte = dimension / 8; + for (long i = 0; i < vectorCount; ++i) { + ByteBuffer byteBuffer = ByteBuffer.allocate((int) dimensionInByte); + random.nextBytes(byteBuffer.array()); + vectors.add(byteBuffer); + } + return vectors; + } + + public static String setIndexParam(int nlist) { + JSONObject indexParam = new JSONObject(); + indexParam.put("nlist", nlist); + return JSONObject.toJSONString(indexParam); + } + + public static String setSearchParam(int nprobe) { + JSONObject searchParam = new JSONObject(); + searchParam.put("nprobe", nprobe); + return JSONObject.toJSONString(searchParam); + } + + public static int getIndexParamValue(String indexParam, String key) { + return JSONObject.parseObject(indexParam).getIntValue(key); + } + + public static JSONObject getCollectionInfo(String collectionInfo) { + return JSONObject.parseObject(collectionInfo); + } + + public static List toListIds(int id) { + List ids = new ArrayList<>(); + ids.add((long)id); + return ids; + } + + public static List toListIds(long id) { + List ids = new ArrayList<>(); + ids.add(id); + return ids; + } +}