mirror of https://github.com/milvus-io/milvus.git
[skip ci] add artifacts (#2750)
* update cron on 0.10.1 Signed-off-by: zw <zw@milvus.io> * enable ivf_pq <ip> search cases on GPU Signed-off-by: zw <zw@milvus.io> * [skip ci] debug Signed-off-by: zw <zw@milvus.io> * [skip ci] add artifacts Signed-off-by: zw <zw@milvus.io> Co-authored-by: zw <zw@milvus.io>pull/2758/head
del-zhenwu
GitHub
parent
cf47007545
commit
002410602e
|
|
@ -160,6 +160,7 @@ pipeline {
|
|||
FORMAT_OS_NAME = "${OS_NAME}".replaceAll("\\.", "-").replaceAll("_", "-")
|
||||
HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${FORMAT_OS_NAME}-${BINARY_VERSION}".toLowerCase()
|
||||
SHARDS_HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-shards-${FORMAT_OS_NAME}-${BINARY_VERSION}".toLowerCase()
|
||||
DEV_TEST_ARTIFACTS = "_artifacts/${FROMAT_SEMVER}/${FORMAT_OS_NAME}"
|
||||
}
|
||||
|
||||
agent {
|
||||
|
|
@ -175,6 +176,7 @@ pipeline {
|
|||
steps {
|
||||
container('milvus-test-env') {
|
||||
script {
|
||||
sh "mkdir -p ${env.DEV_TEST_ARTIFACTS}"
|
||||
boolean isNightlyTest = isTimeTriggeredBuild()
|
||||
if (isNightlyTest) {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy"
|
||||
|
|
@ -192,6 +194,7 @@ pipeline {
|
|||
cleanup {
|
||||
container('milvus-test-env') {
|
||||
script {
|
||||
archiveArtifacts artifacts: "${env.DEV_TEST_ARTIFACTS}/**", allowEmptyArchive: true
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy"
|
||||
boolean isNightlyTest = isTimeTriggeredBuild()
|
||||
if (isNightlyTest) {
|
||||
|
|
|
|||
|
|
@ -26,6 +26,6 @@ timeout(time: 180, unit: 'MINUTES') {
|
|||
|
||||
dir ("tests/milvus_python_test") {
|
||||
sh 'python3 -m pip install -r requirements.txt'
|
||||
sh "pytest . --level=2 --alluredir=\"test_out/dev/shards/\" --ip ${env.SHARDS_HELM_RELEASE_NAME}.milvus.svc.cluster.local"
|
||||
sh "pytest . --level=2 --alluredir=\"test_out/dev/shards/\" --ip ${env.SHARDS_HELM_RELEASE_NAME}.milvus.svc.cluster.local >> ${WORKSPACE}/${env.DEV_TEST_ARTIFACTS}/milvus_shards_dev_test.log"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ timeout(time: 180, unit: 'MINUTES') {
|
|||
dir ("tests/milvus_python_test") {
|
||||
// sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
|
||||
sh 'python3 -m pip install -r requirements.txt'
|
||||
sh "pytest . --level=2 --alluredir=\"test_out/dev/single/mysql\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
|
||||
sh "pytest . --level=2 --alluredir=\"test_out/dev/single/mysql\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local >> ${WORKSPACE}/${env.DEV_TEST_ARTIFACTS}/milvus_mysql_dev_test.log"
|
||||
}
|
||||
// sqlite database backend test
|
||||
load "ci/jenkins/step/cleanupSingleDev.groovy"
|
||||
|
|
@ -54,7 +54,7 @@ timeout(time: 180, unit: 'MINUTES') {
|
|||
}
|
||||
}
|
||||
dir ("tests/milvus_python_test") {
|
||||
sh "pytest . --level=2 --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
|
||||
sh "pytest . --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local --port=19121 --handler=HTTP"
|
||||
sh "pytest . --level=2 --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local >> ${WORKSPACE}/${env.DEV_TEST_ARTIFACTS}/milvus_sqlite_dev_test.log"
|
||||
sh "pytest . --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local --port=19121 --handler=HTTP >> ${WORKSPACE}/${env.DEV_TEST_ARTIFACTS}/milvus_sqlite_http_dev_test.log"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ timeout(time: 120, unit: 'MINUTES') {
|
|||
dir ("tests/milvus_python_test") {
|
||||
// sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
|
||||
sh 'python3 -m pip install -r requirements.txt'
|
||||
sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local --service ${env.HELM_RELEASE_NAME}"
|
||||
sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local --service ${env.HELM_RELEASE_NAME} >> ${WORKSPACE}/${env.DEV_TEST_ARTIFACTS}/milvus_mysql_dev_test.log"
|
||||
// sh "pytest test_restart.py --alluredir=\"test_out/dev/single/mysql\" --level=3 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local --service ${env.HELM_RELEASE_NAME}"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@
|
|||
<dependency>
|
||||
<groupId>io.milvus</groupId>
|
||||
<artifactId>milvus-sdk-java</artifactId>
|
||||
<version>0.8.0-SNAPSHOT</version>
|
||||
<version>0.8.2</version>
|
||||
</dependency>
|
||||
|
||||
<!-- <dependency>-->
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package com;
|
||||
|
||||
import io.milvus.client.*;
|
||||
|
|
@ -7,12 +8,13 @@ import java.util.List;
|
|||
import java.util.concurrent.ForkJoinPool;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public class TestPS {
|
||||
private static int dimension = 128;
|
||||
private static String host = "127.0.0.1";
|
||||
private static String port = "19530";
|
||||
private static int dimension = 512;
|
||||
private static String host = "192.168.1.112";
|
||||
private static String port = "19532";
|
||||
|
||||
public static void setHost(String host) {
|
||||
TestPS.host = host;
|
||||
|
|
@ -26,16 +28,16 @@ public class TestPS {
|
|||
|
||||
public static void main(String[] args) throws ConnectFailedException {
|
||||
int nb = 10000;
|
||||
int nq = 5;
|
||||
int nprobe = 32;
|
||||
int top_k = 10;
|
||||
int loops = 100000;
|
||||
int nq = 1;
|
||||
int nprobe = 1024;
|
||||
int top_k = 2;
|
||||
int loops = 100000000;
|
||||
// int index_file_size = 1024;
|
||||
String collectionName = "sift_1b_2048_128_l2";
|
||||
String collectionName = "random_1m_2048_512_ip_sq8";
|
||||
|
||||
|
||||
List<List<Float>> vectors = Utils.genVectors(nb, dimension, true);
|
||||
List<List<Float>> queryVectors = vectors.subList(0, nq);
|
||||
|
||||
|
||||
CommandLineParser parser = new DefaultParser();
|
||||
Options options = new Options();
|
||||
|
|
@ -63,32 +65,69 @@ public class TestPS {
|
|||
.withPort(Integer.parseInt(port))
|
||||
.build();
|
||||
client.connect(connectParam);
|
||||
|
||||
// String collectionName = RandomStringUtils.randomAlphabetic(10);
|
||||
// TableSchema tableSchema = new TableSchema.Builder(collectionName, dimension)
|
||||
// .withIndexFileSize(index_file_size)
|
||||
// .withMetricType(MetricType.IP)
|
||||
// .build();
|
||||
// Response res = client.createTable(tableSchema);
|
||||
List<Long> vectorIds;
|
||||
vectorIds = Stream.iterate(0L, n -> n)
|
||||
.limit(nb)
|
||||
.collect(Collectors.toList());
|
||||
InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).withVectorIds(vectorIds).build();
|
||||
// List<Long> vectorIds;
|
||||
// vectorIds = Stream.iterate(0L, n -> n)
|
||||
// .limit(nb)
|
||||
// .collect(Collectors.toList());
|
||||
// InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(vectors).withVectorIds(vectorIds).build();
|
||||
System.setProperty("java.util.concurrent.ForkJoinPool.common.parallelism", "50");
|
||||
ForkJoinPool executor_search = new ForkJoinPool();
|
||||
for (int i = 0; i < loops; i++) {
|
||||
executor_search.execute(
|
||||
() -> {
|
||||
InsertResponse res_insert = client.insert(insertParam);
|
||||
assert (res_insert.getResponse().ok());
|
||||
System.out.println("In insert");
|
||||
SearchParam searchParam = new SearchParam.Builder(collectionName).withFloatVectors(queryVectors).withTopK(top_k).build();
|
||||
// for (int i = 0; i < loops; i++) {
|
||||
// List<List<Float>> queryVectors = Utils.genVectors(nq, dimension, true);
|
||||
// executor_search.execute(
|
||||
// () -> {
|
||||
//// InsertResponse res_insert = client.insert(insertParam);
|
||||
//// assert (res_insert.getResponse().ok());
|
||||
//// System.out.println("In insert");
|
||||
// String params = "{\"nprobe\":1024}";
|
||||
// SearchParam searchParam = new SearchParam.Builder(collectionName)
|
||||
// .withFloatVectors(queryVectors)
|
||||
// .withParamsInJson(params)
|
||||
// .withTopK(top_k).build();
|
||||
// SearchResponse res_search = client.search(searchParam);
|
||||
// assert (res_search.getResponse().ok());
|
||||
// });
|
||||
// }
|
||||
|
||||
IntStream.range(0, loops).parallel().forEach(index -> {
|
||||
List<List<Float>> queryVectors = Utils.genVectors(nq, dimension, true);
|
||||
String params = "{\"nprobe\":1024}";
|
||||
SearchParam searchParam = new SearchParam.Builder(collectionName)
|
||||
.withFloatVectors(queryVectors)
|
||||
.withParamsInJson(params)
|
||||
.withTopK(top_k).build();
|
||||
SearchResponse res_search = client.search(searchParam);
|
||||
assert (res_search.getResponse().ok());
|
||||
});
|
||||
}
|
||||
executor_search.awaitQuiescence(300, TimeUnit.SECONDS);
|
||||
executor_search.shutdown();
|
||||
CountEntitiesResponse getTableRowCountResponse = client.countEntities(collectionName);
|
||||
System.out.println(getTableRowCountResponse.getCollectionEntityCount());
|
||||
|
||||
// int thread_num = 50;
|
||||
// ForkJoinPool executor = new ForkJoinPool();
|
||||
// for (int i = 0; i < thread_num; i++) {
|
||||
// executor.execute(
|
||||
// () -> {
|
||||
// String params = "{\"nprobe\":\"1024\"}";
|
||||
// SearchParam searchParam = new SearchParam.Builder(collectionName)
|
||||
// .withFloatVectors(queryVectors)
|
||||
// .withParamsInJson(params)
|
||||
// .withTopK(top_k).build();
|
||||
// SearchResponse res_search = client.search(searchParam);
|
||||
// assert (res_search.getResponse().ok());
|
||||
// });
|
||||
// }
|
||||
// executor.awaitQuiescence(100, TimeUnit.SECONDS);
|
||||
// executor.shutdown();
|
||||
// CountEntitiesResponse getTableRowCountResponse = client.countEntities(collectionName);
|
||||
// System.out.println(getTableRowCountResponse.getCollectionEntityCount());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
import pdb
|
||||
import struct
|
||||
from random import sample
|
||||
|
||||
import pytest
|
||||
import threading
|
||||
import datetime
|
||||
import logging
|
||||
from time import sleep
|
||||
import concurrent.futures
|
||||
from multiprocessing import Process
|
||||
import pytest
|
||||
import numpy
|
||||
import sklearn.preprocessing
|
||||
from milvus import IndexType, MetricType
|
||||
|
|
@ -802,28 +802,32 @@ class TestSearchBase:
|
|||
distance_1 = numpy.inner(numpy.array(query_vecs[0]), numpy.array(vectors[1]))
|
||||
assert abs(result[0][0].distance - max(distance_0, distance_1)) <= gen_inaccuracy(result[0][0].distance)
|
||||
|
||||
# TODO: enable
|
||||
# @pytest.mark.repeat(5)
|
||||
@pytest.mark.timeout(30)
|
||||
def _test_search_concurrent(self, connect, collection):
|
||||
vectors, ids = self.init_data(connect, collection)
|
||||
thread_num = 10
|
||||
nb = 100
|
||||
top_k = 10
|
||||
threads = []
|
||||
query_vecs = vectors[nb//2:nb]
|
||||
def search():
|
||||
status, result = connect.search(collection, top_k, query_vecs)
|
||||
assert len(result) == len(query_vecs)
|
||||
for i in range(len(query_vecs)):
|
||||
assert result[i][0].id in ids
|
||||
assert result[i][0].distance == 0.0
|
||||
for i in range(thread_num):
|
||||
x = threading.Thread(target=search, args=())
|
||||
threads.append(x)
|
||||
x.start()
|
||||
for th in threads:
|
||||
th.join()
|
||||
# def test_search_concurrent(self, connect, collection):
|
||||
# vectors, ids = self.init_data(connect, collection, nb=5000)
|
||||
# thread_num = 50
|
||||
# nq = 1
|
||||
# top_k = 2
|
||||
# threads = []
|
||||
# query_vecs = vectors[:nq]
|
||||
# def search(thread_number):
|
||||
# for i in range(1000000):
|
||||
# status, result = connect.search(collection, top_k, query_vecs, timeout=2)
|
||||
# assert len(result) == len(query_vecs)
|
||||
# assert status.OK()
|
||||
# if i % 1000 == 0:
|
||||
# logging.getLogger().info("In %d, %d" % (thread_number, i))
|
||||
# logging.getLogger().info("%d finished" % thread_number)
|
||||
# # with concurrent.futures.ThreadPoolExecutor(max_workers=thread_num) as executor:
|
||||
# # future_results = {executor.submit(
|
||||
# # search): i for i in range(1000000)}
|
||||
# # for future in concurrent.futures.as_completed(future_results):
|
||||
# # future.result()
|
||||
# for i in range(thread_num):
|
||||
# t = threading.Thread(target=search, args=(i, ))
|
||||
# threads.append(t)
|
||||
# t.start()
|
||||
# for t in threads:
|
||||
# t.join()
|
||||
|
||||
@pytest.mark.level(2)
|
||||
@pytest.mark.timeout(30)
|
||||
|
|
|
|||
Loading…
Reference in New Issue