From 63023f4d5df34e811c5afd534a47c8fc680a1214 Mon Sep 17 00:00:00 2001 From: starlord Date: Tue, 15 Oct 2019 18:57:18 +0800 Subject: [PATCH 01/36] remove zilliz Former-commit-id: ef67fdeecac6964fc11ac1d17c7898423dfb4430 --- core/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main.cpp b/core/src/main.cpp index 769483acd3..4178626f32 100644 --- a/core/src/main.cpp +++ b/core/src/main.cpp @@ -50,7 +50,7 @@ print_banner() { std::cout << " / /|_/ // // /_| |/ / /_/ /\\ \\ " << std::endl; std::cout << " /_/ /_/___/____/___/\\____/___/ " << std::endl; std::cout << std::endl; - std::cout << "Welcome to use Milvus by Zilliz!" << std::endl; + std::cout << "Welcome to use Milvus!" << std::endl; std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME << std::endl; std::cout << std::endl; } From 2e1c9a1b88fbc7b946928913d72ea9c75d38c527 Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Mon, 16 Sep 2019 14:43:44 +0800 Subject: [PATCH 02/36] Initial commit Former-commit-id: 9d3ec9d21264217e4481867bebb27c263a264912 --- LICENSE | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 1 + 2 files changed, 202 insertions(+) create mode 100644 LICENSE create mode 100644 README.md diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000000..d18292fd99 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# milvus \ No newline at end of file From 1a6e3f048eeea0485571a0a7616f633c7bb542b9 Mon Sep 17 00:00:00 2001 From: jinhai Date: Tue, 17 Sep 2019 11:18:02 +0800 Subject: [PATCH 03/36] Update README.md Former-commit-id: 7b168bf104625554cf6849cc31c4f5f4bbccfc8f --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d18292fd99..f496cdd884 100644 --- a/README.md +++ b/README.md @@ -1 +1 @@ -# milvus \ No newline at end of file +# Milvus is coming soon! \ No newline at end of file From f63880a613d8322fd3eb1b16a91755f5199c4214 Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Wed, 18 Sep 2019 11:42:17 +0800 Subject: [PATCH 04/36] Update issue templates Former-commit-id: d4798b66bf309331a5d4ecdd5ce20103a9ff60e1 --- .github/ISSUE_TEMPLATE/bug_report.md | 29 +++++++++++++++ .../ISSUE_TEMPLATE/documentation-request.md | 35 +++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 20 +++++++++++ .github/ISSUE_TEMPLATE/general-question.md | 10 ++++++ 4 files changed, 94 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/documentation-request.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/general-question.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000000..8d2a7cbfc4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,29 @@ +--- +name: Bug report +about: Create a bug report to help us improve Milvus +title: "[BUG]" +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**Steps/Code to reproduce behavior** +Follow this [guide](http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) to craft a minimal bug report. This helps us reproduce the issue you're having and resolve the issue more quickly. + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Environment details** +- Hardware/Softward conditions (OS, CPU, GPU, Memory) +- Method of installation (Docker, or from source) +- Milvus version (v0.3.1, or v0.4.0) +- Milvus configuration (Settings you made in `server_config.yaml`) + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/documentation-request.md b/.github/ISSUE_TEMPLATE/documentation-request.md new file mode 100644 index 0000000000..91a17502ef --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation-request.md @@ -0,0 +1,35 @@ +--- +name: Documentation request +about: Report incorrect or needed documentation +title: "[DOC]" +labels: '' +assignees: '' + +--- + +## Report incorrect documentation + +**Location of incorrect documentation** +Provide links and line numbers if applicable. + +**Describe the problems or issues found in the documentation** +A clear and concise description of what you found to be incorrect. + +**Steps taken to verify documentation is incorrect** +List any steps you have taken: + +**Suggested fix for documentation** +Detail proposed changes to fix the documentation if you have any. + +--- + +## Report needed documentation + +**Report needed documentation** +A clear and concise description of what documentation you believe it is needed and why. + +**Describe the documentation you'd like** +A clear and concise description of what you want to happen. + +**Steps taken to search for needed documentation** +List any steps you have taken: diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000000..d08948c4ed --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for Milvus +title: "[FEATURE]" +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. E.g. I wish I could use Milvus to do [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context, code examples, or references to existing implementations about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/general-question.md b/.github/ISSUE_TEMPLATE/general-question.md new file mode 100644 index 0000000000..8e59ee063a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/general-question.md @@ -0,0 +1,10 @@ +--- +name: General question +about: Ask a general question about Milvus +title: "[QUESTION]" +labels: '' +assignees: '' + +--- + +**What is your question?** From 2892058e490c16c7eb91963186abb81138e3a29e Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Wed, 25 Sep 2019 16:42:32 +0800 Subject: [PATCH 05/36] Update issue templates Former-commit-id: f2a6152a0a589dadc7ea94569ea312f1e5aac41a --- .github/ISSUE_TEMPLATE/-bug--bug-report.md | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/-bug--bug-report.md diff --git a/.github/ISSUE_TEMPLATE/-bug--bug-report.md b/.github/ISSUE_TEMPLATE/-bug--bug-report.md new file mode 100644 index 0000000000..086463d2d5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/-bug--bug-report.md @@ -0,0 +1,29 @@ +--- +name: ":bug: Bug report" +about: Create a bug report to help us improve Milvus +title: "[BUG]" +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**Steps/Code to reproduce behavior** +Follow this [guide](http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) to craft a minimal bug report. This helps us reproduce the issue you're having and resolve the issue more quickly. + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Environment details** +- Hardware/Softward conditions (OS, CPU, GPU, Memory) +- Method of installation (Docker, or from source) +- Milvus version (v0.3.1, or v0.4.0) +- Milvus configuration (Settings you made in `server_config.yaml`) + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Additional context** +Add any other context about the problem here. From ada2c0b15ddb3955d396b6c9b1c158a7dc26a31e Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Wed, 25 Sep 2019 16:44:56 +0800 Subject: [PATCH 06/36] Update bug_report.md Former-commit-id: e63f01173cccc8b0f58f880bfda59b6cf5bafff8 --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 8d2a7cbfc4..93ea941565 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,5 +1,5 @@ --- -name: Bug report +name: :bug:Bug report about: Create a bug report to help us improve Milvus title: "[BUG]" labels: '' From 8364fabe24f6e6f5ea15d89a504342b1d9a020a6 Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Wed, 25 Sep 2019 16:45:12 +0800 Subject: [PATCH 07/36] Update bug_report.md Former-commit-id: 6485d40c6f4852ea0bb4b7ee72f3d28b712eb176 --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 93ea941565..647526fdbc 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,5 +1,5 @@ --- -name: :bug:Bug report +name: :bug: Bug report about: Create a bug report to help us improve Milvus title: "[BUG]" labels: '' From 4599df8920a373c3ccf0c499a869128ed3351f48 Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Wed, 25 Sep 2019 16:45:37 +0800 Subject: [PATCH 08/36] Update bug_report.md Former-commit-id: 6222f7d20d5de6764985609accf6db6a5d1edcee --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 647526fdbc..8d2a7cbfc4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,5 +1,5 @@ --- -name: :bug: Bug report +name: Bug report about: Create a bug report to help us improve Milvus title: "[BUG]" labels: '' From 6d3ca031e286ed01a2d81a16f2edb37320626936 Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Wed, 25 Sep 2019 16:54:37 +0800 Subject: [PATCH 09/36] Update issue templates Former-commit-id: 900b0275217b7c66110fcdc8d7fea14ae94222ef --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 8d2a7cbfc4..0c9d1aa742 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,7 +1,7 @@ --- name: Bug report about: Create a bug report to help us improve Milvus -title: "[BUG]" +title: ":bug:[BUG]" labels: '' assignees: '' From 2b8e8e8465dbd24a3ea69ac9a51f081cdf1c8cf5 Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Wed, 25 Sep 2019 16:56:02 +0800 Subject: [PATCH 10/36] Update issue templates Former-commit-id: f4f48f50c508700df5f98df3de4f2059e6d2a915 --- .github/ISSUE_TEMPLATE/-bug--bug-report.md | 29 ---------------------- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- 2 files changed, 1 insertion(+), 30 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/-bug--bug-report.md diff --git a/.github/ISSUE_TEMPLATE/-bug--bug-report.md b/.github/ISSUE_TEMPLATE/-bug--bug-report.md deleted file mode 100644 index 086463d2d5..0000000000 --- a/.github/ISSUE_TEMPLATE/-bug--bug-report.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -name: ":bug: Bug report" -about: Create a bug report to help us improve Milvus -title: "[BUG]" -labels: '' -assignees: '' - ---- - -**Describe the bug** -A clear and concise description of what the bug is. - -**Steps/Code to reproduce behavior** -Follow this [guide](http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) to craft a minimal bug report. This helps us reproduce the issue you're having and resolve the issue more quickly. - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Environment details** -- Hardware/Softward conditions (OS, CPU, GPU, Memory) -- Method of installation (Docker, or from source) -- Milvus version (v0.3.1, or v0.4.0) -- Milvus configuration (Settings you made in `server_config.yaml`) - -**Screenshots** -If applicable, add screenshots to help explain your problem. - -**Additional context** -Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 0c9d1aa742..2cf3ee07ee 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,7 +1,7 @@ --- name: Bug report about: Create a bug report to help us improve Milvus -title: ":bug:[BUG]" +title: ":bug: [BUG]" labels: '' assignees: '' From 59f745f1ad109a2e372d6e80654205b0069945f6 Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Wed, 25 Sep 2019 16:57:51 +0800 Subject: [PATCH 11/36] Update issue templates Former-commit-id: d80503f33031e122bcaa8c3dcb173bf263ecedfd --- .github/ISSUE_TEMPLATE/bug-report.md | 29 ++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug-report.md diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 0000000000..8d2a7cbfc4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,29 @@ +--- +name: Bug report +about: Create a bug report to help us improve Milvus +title: "[BUG]" +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**Steps/Code to reproduce behavior** +Follow this [guide](http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) to craft a minimal bug report. This helps us reproduce the issue you're having and resolve the issue more quickly. + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Environment details** +- Hardware/Softward conditions (OS, CPU, GPU, Memory) +- Method of installation (Docker, or from source) +- Milvus version (v0.3.1, or v0.4.0) +- Milvus configuration (Settings you made in `server_config.yaml`) + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Additional context** +Add any other context about the problem here. From 15dcc8055ac141de97a33853c5d81724570f1126 Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Wed, 25 Sep 2019 16:59:00 +0800 Subject: [PATCH 12/36] Delete bug-report.md Former-commit-id: f804d0b7bd80bb688f2dc4f442efc86ceb999986 --- .github/ISSUE_TEMPLATE/bug-report.md | 29 ---------------------------- 1 file changed, 29 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/bug-report.md diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md deleted file mode 100644 index 8d2a7cbfc4..0000000000 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -name: Bug report -about: Create a bug report to help us improve Milvus -title: "[BUG]" -labels: '' -assignees: '' - ---- - -**Describe the bug** -A clear and concise description of what the bug is. - -**Steps/Code to reproduce behavior** -Follow this [guide](http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) to craft a minimal bug report. This helps us reproduce the issue you're having and resolve the issue more quickly. - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Environment details** -- Hardware/Softward conditions (OS, CPU, GPU, Memory) -- Method of installation (Docker, or from source) -- Milvus version (v0.3.1, or v0.4.0) -- Milvus configuration (Settings you made in `server_config.yaml`) - -**Screenshots** -If applicable, add screenshots to help explain your problem. - -**Additional context** -Add any other context about the problem here. From 930cfa11744fb5c4275a3458fc88f1984db3ba2b Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Wed, 25 Sep 2019 17:00:37 +0800 Subject: [PATCH 13/36] Update issue templates Former-commit-id: a4c52b57b55c9b4f0ce3c5a34925e18cbb6f1a90 --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 2cf3ee07ee..8d2a7cbfc4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,7 +1,7 @@ --- name: Bug report about: Create a bug report to help us improve Milvus -title: ":bug: [BUG]" +title: "[BUG]" labels: '' assignees: '' From decb49ae98b4f93c4ef70959713527dea431501a Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Wed, 25 Sep 2019 18:03:59 +0800 Subject: [PATCH 14/36] Update bug_report.md Former-commit-id: b4e2f87d7da26cd6200284bb59f3c4fc71bffd4e --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 8d2a7cbfc4..d3a9f247d6 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,5 +1,5 @@ --- -name: Bug report +name: "\U0001F41B Bug report" about: Create a bug report to help us improve Milvus title: "[BUG]" labels: '' From c5781cdb6eaa478a7b2f8e0730d74c5f4a5e938d Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Wed, 25 Sep 2019 18:05:10 +0800 Subject: [PATCH 15/36] Update general-question.md Former-commit-id: feea1e94c43772d191f5e442c566d8cfcf8ba2a3 --- .github/ISSUE_TEMPLATE/general-question.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/general-question.md b/.github/ISSUE_TEMPLATE/general-question.md index 8e59ee063a..32ce5dd701 100644 --- a/.github/ISSUE_TEMPLATE/general-question.md +++ b/.github/ISSUE_TEMPLATE/general-question.md @@ -1,5 +1,5 @@ --- -name: General question +name: "\U0001F914 General question" about: Ask a general question about Milvus title: "[QUESTION]" labels: '' From 6c1c20ee26a1933f7b81b6e67e8b1862434774b6 Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Wed, 25 Sep 2019 18:05:54 +0800 Subject: [PATCH 16/36] Update feature_request.md Former-commit-id: b432bf29ac91eaf8f9e606df12245f421294985c --- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index d08948c4ed..24de651b12 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,5 +1,5 @@ --- -name: Feature request +name: "\U0001F680 Feature request" about: Suggest an idea for Milvus title: "[FEATURE]" labels: '' From 98a1befbed1d57c9935ad604dff959c6795f755e Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Wed, 25 Sep 2019 18:09:20 +0800 Subject: [PATCH 17/36] Update documentation-request.md Former-commit-id: 5beb7a1233c97a30fc64435556582c2a587ba728 --- .github/ISSUE_TEMPLATE/documentation-request.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/documentation-request.md b/.github/ISSUE_TEMPLATE/documentation-request.md index 91a17502ef..1e3193f9f3 100644 --- a/.github/ISSUE_TEMPLATE/documentation-request.md +++ b/.github/ISSUE_TEMPLATE/documentation-request.md @@ -1,5 +1,5 @@ --- -name: Documentation request +name: "\U0001F4DD Documentation request" about: Report incorrect or needed documentation title: "[DOC]" labels: '' From d6496280a331a7c717b40ce152a5c362f004feeb Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Tue, 15 Oct 2019 18:42:20 +0800 Subject: [PATCH 18/36] Update README.md Former-commit-id: 7eb9bb12d55d7c2cd80cc515631cdabe2f5b9102 --- README.md | 248 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 247 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f496cdd884..7a3963353e 100644 --- a/README.md +++ b/README.md @@ -1 +1,247 @@ -# Milvus is coming soon! \ No newline at end of file +- [Slack Community](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk) +- [Blog](https://www.milvus.io/blog/) + +# Welcome to Milvus + +Firstly, welcome, and thanks for your interest in [Milvus](https://milvus.io)! No matter who you are, what you do, we greatly appreciate your contribution to help us reinvent data science with Milvus. + +## What is Milvus + +Milvus is an open source vector search engine that supports similarity search of large-scale vectors. Built on optimized indexing algorithm, it is compatible with major AI/ML models. + +Milvus was developed by ZILLIZ, a tech startup that intends to reinvent data science, with the purpose of providing enterprises with efficient and scalable similarity search and analysis of feature vectors and unstructured data. + +Milvus provides stable Python, C++ and Java APIs. + +Keep up-to-date with newest releases and latest updates by reading Milvus [release notes](https://milvus.io/docs/en/Releases/v0.4.0/). + +- GPU-accelerated search engine + + Milvus is designed for the largest scale of vector index. CPU/GPU heterogeneous computing architecture allows you to process data at a speed 1000 times faster. + +- Intelligent index + + With a “Decide Your Own Algorithm” approach, you can embed machine learning and advanced algorithms into Milvus without the headache of complex data engineering or migrating data between disparate systems. Milvus is built on optimized indexing algorithm based on quantization indexing, tree-based and graph indexing methods. + +- Strong scalability + + The data is stored and computed on a distributed architecture. This lets you scale data sizes up and down without redesigning the system. + +## Architecture +![Milvus_arch](https://milvus.io/docs/assets/milvus_arch.png) + +## Get started + +### Install and start Milvus server + +#### Use Docker + +Use Docker to install Milvus is a breeze. See the [Milvus install guide](https://milvus.io/docs/en/userguide/install_milvus/) for details. + +#### Use source code + +##### Compilation + +###### Step 1 Install necessary tools + +```shell +# Install tools +Centos7 : +$ yum install gfortran qt4 flex bison +$ yum install mysql-devel mysql + +Ubuntu 16.04 or 18.04: +$ sudo apt-get install gfortran qt4-qmake flex bison +$ sudo apt-get install libmysqlclient-dev mysql-client +``` + +Verify the existence of `libmysqlclient_r.so`: + +```shell +# Verify existence +$ locate libmysqlclient_r.so +``` + +If not, you need to create a symbolic link: + +```shell +# Locate libmysqlclient.so +$ sudo updatedb +$ locate libmysqlclient.so + +# Create symbolic link +$ sudo ln -s /path/to/libmysqlclient.so /path/to/libmysqlclient_r.so +``` + +###### Step 2 Build + +```shell +$ cd [Milvus sourcecode path]/core +$ ./build.sh -t Debug +or +$ ./build.sh -t Release +``` + +When the build is completed, all the stuff that you need in order to run Milvus will be installed under `[Milvus root path]/core/milvus`. + +If you encounter the following error message, +`protocol https not supported or disabled in libcurl` + +please reinstall CMake with curl: + +1. Install curl development files: + ```shell + CentOS 7: + $ yum install curl-devel + Ubuntu 16.04 or 18.04: + $ sudo apt-get install libcurl4-openssl-dev + ``` + +2. Install [CMake 3.14](https://github.com/Kitware/CMake/releases/download/v3.14.6/cmake-3.14.6.tar.gz): + ```shell + $ ./bootstrap --system-curl + $ make + $ sudo make install + ``` + +##### code format and linting +Install clang-format and clang-tidy +```shell +CentOS 7: +$ yum install clang +Ubuntu 16.04: +$ sudo apt-get install clang-tidy +$ sudo su +$ wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - +$ apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main" +$ apt-get update +$ apt-get install clang-format-6.0 +Ubuntu 18.04: +$ sudo apt-get install clang-tidy clang-format + +$ rm cmake_build/CMakeCache.txt +``` +Check code style +```shell +$ ./build.sh -l +``` +To format the code +```shell +$ cd cmake_build +$ make clang-format +``` + +##### Run unit test + +```shell +$ ./build.sh -u +``` + +##### Run code coverage +Install lcov +```shell +CentOS 7: +$ yum install lcov +Ubuntu 16.04 or 18.04: +$ sudo apt-get install lcov +``` +```shell +$ ./build.sh -u -c +``` +Run mysql docker +```shell +docker pull mysql:latest +docker run -p 3306:3306 -e MYSQL_ROOT_PASSWORD=123456 -d mysql:latest +``` +Run code coverage +```shell +$ ./coverage.sh -u root -p 123456 -t 127.0.0.1 +``` + +##### Launch Milvus server + +```shell +$ cd [Milvus root path]/core/milvus +``` + +Add `lib/` directory to `LD_LIBRARY_PATH` + +``` +$ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/path/to/milvus/lib +``` + +Then start Milvus server: + +``` +$ cd scripts +$ ./start_server.sh +``` + +To stop Milvus server, run: + +```shell +$ ./stop_server.sh +``` + +To edit Milvus settings in `conf/server_config.yaml` and `conf/log_config.conf`, please read [Milvus Configuration](https://www.milvus-io/docs/master/reference/milvus_config.md). + +### Try your first Milvus program + +#### Run Python example code + +Make sure [Python 3.4](https://www.python.org/downloads/) or higher is already installed and in use. + +Install Milvus Python SDK. + +```shell +# Install Milvus Python SDK +$ pip install pymilvus==0.2.0 +``` + +Create a new file `example.py`, and add [Python example code](https://github.com/milvus-io/pymilvus/blob/branch-0.3.1/examples/AdvancedExample.py) to it. + +Run the example code. + +```python +# Run Milvus Python example +$ python3 example.py +``` + +#### Run C++ example code + +```shell + # Run Milvus C++ example + $ cd [Milvus root path]/core/milvus/bin + $ ./sdk_simple +``` + +## Contribution guidelines + +Contributions are welcomed and greatly appreciated. If you want to contribute to Milvus, please read our [contribution guidelines](CONTRIBUTING.md). This project adheres to the [code of conduct](CODE OF CONDUCT.md) of Milvus. By participating, you are expected to uphold this code. + +We use [GitHub issues](https://github.com/milvus-io/milvus/issues) to track issues and bugs. For general questions and public discussions, please join our community. + +## Join the Milvus community + +To connect with other users and contributors, welcome to join our [slack channel](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk). + +## Milvus Roadmap + +Please read our [roadmap](https://milvus.io/docs/en/roadmap/) to learn about upcoming features. + +## Resources + +[Milvus official website](https://www.milvus.io) + +[Milvus docs](https://www.milvus.io/docs/en/QuickStart/) + +[Milvus blog](https://www.milvus.io/blog/) + +[Milvus CSDN](https://mp.csdn.net/mdeditor/100041006#) + +[Milvus roadmap](https://milvus.io/docs/en/roadmap/) + + +## License + +[Apache 2.0 license](milvus-io/milvus/LICENSE.md) \ No newline at end of file From 498ef8ecb804dc7b97547c3a2de14d145b4dc14a Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Tue, 15 Oct 2019 18:43:51 +0800 Subject: [PATCH 19/36] Update Documents Former-commit-id: 04f2966a1dec3b28f33e06037551631fd5f49b6d --- core/CHANGELOG.md => CHANGELOG.md | 0 core/CODE_OF_CONDUCT.md => CODE_OF_CONDUCT.md | 0 core/CONTRIBUTING.md => CONTRIBUTING.md | 0 core/NOTICE.md => NOTICE.md | 0 core/README.md | 247 ------------------ 5 files changed, 247 deletions(-) rename core/CHANGELOG.md => CHANGELOG.md (100%) rename core/CODE_OF_CONDUCT.md => CODE_OF_CONDUCT.md (100%) rename core/CONTRIBUTING.md => CONTRIBUTING.md (100%) rename core/NOTICE.md => NOTICE.md (100%) delete mode 100644 core/README.md diff --git a/core/CHANGELOG.md b/CHANGELOG.md similarity index 100% rename from core/CHANGELOG.md rename to CHANGELOG.md diff --git a/core/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md similarity index 100% rename from core/CODE_OF_CONDUCT.md rename to CODE_OF_CONDUCT.md diff --git a/core/CONTRIBUTING.md b/CONTRIBUTING.md similarity index 100% rename from core/CONTRIBUTING.md rename to CONTRIBUTING.md diff --git a/core/NOTICE.md b/NOTICE.md similarity index 100% rename from core/NOTICE.md rename to NOTICE.md diff --git a/core/README.md b/core/README.md deleted file mode 100644 index 7a3963353e..0000000000 --- a/core/README.md +++ /dev/null @@ -1,247 +0,0 @@ -- [Slack Community](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk) -- [Blog](https://www.milvus.io/blog/) - -# Welcome to Milvus - -Firstly, welcome, and thanks for your interest in [Milvus](https://milvus.io)! No matter who you are, what you do, we greatly appreciate your contribution to help us reinvent data science with Milvus. - -## What is Milvus - -Milvus is an open source vector search engine that supports similarity search of large-scale vectors. Built on optimized indexing algorithm, it is compatible with major AI/ML models. - -Milvus was developed by ZILLIZ, a tech startup that intends to reinvent data science, with the purpose of providing enterprises with efficient and scalable similarity search and analysis of feature vectors and unstructured data. - -Milvus provides stable Python, C++ and Java APIs. - -Keep up-to-date with newest releases and latest updates by reading Milvus [release notes](https://milvus.io/docs/en/Releases/v0.4.0/). - -- GPU-accelerated search engine - - Milvus is designed for the largest scale of vector index. CPU/GPU heterogeneous computing architecture allows you to process data at a speed 1000 times faster. - -- Intelligent index - - With a “Decide Your Own Algorithm” approach, you can embed machine learning and advanced algorithms into Milvus without the headache of complex data engineering or migrating data between disparate systems. Milvus is built on optimized indexing algorithm based on quantization indexing, tree-based and graph indexing methods. - -- Strong scalability - - The data is stored and computed on a distributed architecture. This lets you scale data sizes up and down without redesigning the system. - -## Architecture -![Milvus_arch](https://milvus.io/docs/assets/milvus_arch.png) - -## Get started - -### Install and start Milvus server - -#### Use Docker - -Use Docker to install Milvus is a breeze. See the [Milvus install guide](https://milvus.io/docs/en/userguide/install_milvus/) for details. - -#### Use source code - -##### Compilation - -###### Step 1 Install necessary tools - -```shell -# Install tools -Centos7 : -$ yum install gfortran qt4 flex bison -$ yum install mysql-devel mysql - -Ubuntu 16.04 or 18.04: -$ sudo apt-get install gfortran qt4-qmake flex bison -$ sudo apt-get install libmysqlclient-dev mysql-client -``` - -Verify the existence of `libmysqlclient_r.so`: - -```shell -# Verify existence -$ locate libmysqlclient_r.so -``` - -If not, you need to create a symbolic link: - -```shell -# Locate libmysqlclient.so -$ sudo updatedb -$ locate libmysqlclient.so - -# Create symbolic link -$ sudo ln -s /path/to/libmysqlclient.so /path/to/libmysqlclient_r.so -``` - -###### Step 2 Build - -```shell -$ cd [Milvus sourcecode path]/core -$ ./build.sh -t Debug -or -$ ./build.sh -t Release -``` - -When the build is completed, all the stuff that you need in order to run Milvus will be installed under `[Milvus root path]/core/milvus`. - -If you encounter the following error message, -`protocol https not supported or disabled in libcurl` - -please reinstall CMake with curl: - -1. Install curl development files: - ```shell - CentOS 7: - $ yum install curl-devel - Ubuntu 16.04 or 18.04: - $ sudo apt-get install libcurl4-openssl-dev - ``` - -2. Install [CMake 3.14](https://github.com/Kitware/CMake/releases/download/v3.14.6/cmake-3.14.6.tar.gz): - ```shell - $ ./bootstrap --system-curl - $ make - $ sudo make install - ``` - -##### code format and linting -Install clang-format and clang-tidy -```shell -CentOS 7: -$ yum install clang -Ubuntu 16.04: -$ sudo apt-get install clang-tidy -$ sudo su -$ wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - -$ apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main" -$ apt-get update -$ apt-get install clang-format-6.0 -Ubuntu 18.04: -$ sudo apt-get install clang-tidy clang-format - -$ rm cmake_build/CMakeCache.txt -``` -Check code style -```shell -$ ./build.sh -l -``` -To format the code -```shell -$ cd cmake_build -$ make clang-format -``` - -##### Run unit test - -```shell -$ ./build.sh -u -``` - -##### Run code coverage -Install lcov -```shell -CentOS 7: -$ yum install lcov -Ubuntu 16.04 or 18.04: -$ sudo apt-get install lcov -``` -```shell -$ ./build.sh -u -c -``` -Run mysql docker -```shell -docker pull mysql:latest -docker run -p 3306:3306 -e MYSQL_ROOT_PASSWORD=123456 -d mysql:latest -``` -Run code coverage -```shell -$ ./coverage.sh -u root -p 123456 -t 127.0.0.1 -``` - -##### Launch Milvus server - -```shell -$ cd [Milvus root path]/core/milvus -``` - -Add `lib/` directory to `LD_LIBRARY_PATH` - -``` -$ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/path/to/milvus/lib -``` - -Then start Milvus server: - -``` -$ cd scripts -$ ./start_server.sh -``` - -To stop Milvus server, run: - -```shell -$ ./stop_server.sh -``` - -To edit Milvus settings in `conf/server_config.yaml` and `conf/log_config.conf`, please read [Milvus Configuration](https://www.milvus-io/docs/master/reference/milvus_config.md). - -### Try your first Milvus program - -#### Run Python example code - -Make sure [Python 3.4](https://www.python.org/downloads/) or higher is already installed and in use. - -Install Milvus Python SDK. - -```shell -# Install Milvus Python SDK -$ pip install pymilvus==0.2.0 -``` - -Create a new file `example.py`, and add [Python example code](https://github.com/milvus-io/pymilvus/blob/branch-0.3.1/examples/AdvancedExample.py) to it. - -Run the example code. - -```python -# Run Milvus Python example -$ python3 example.py -``` - -#### Run C++ example code - -```shell - # Run Milvus C++ example - $ cd [Milvus root path]/core/milvus/bin - $ ./sdk_simple -``` - -## Contribution guidelines - -Contributions are welcomed and greatly appreciated. If you want to contribute to Milvus, please read our [contribution guidelines](CONTRIBUTING.md). This project adheres to the [code of conduct](CODE OF CONDUCT.md) of Milvus. By participating, you are expected to uphold this code. - -We use [GitHub issues](https://github.com/milvus-io/milvus/issues) to track issues and bugs. For general questions and public discussions, please join our community. - -## Join the Milvus community - -To connect with other users and contributors, welcome to join our [slack channel](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk). - -## Milvus Roadmap - -Please read our [roadmap](https://milvus.io/docs/en/roadmap/) to learn about upcoming features. - -## Resources - -[Milvus official website](https://www.milvus.io) - -[Milvus docs](https://www.milvus.io/docs/en/QuickStart/) - -[Milvus blog](https://www.milvus.io/blog/) - -[Milvus CSDN](https://mp.csdn.net/mdeditor/100041006#) - -[Milvus roadmap](https://milvus.io/docs/en/roadmap/) - - -## License - -[Apache 2.0 license](milvus-io/milvus/LICENSE.md) \ No newline at end of file From aece3eb1e10fdf229eb9f9a9e0c5f8a2fab332e9 Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Tue, 15 Oct 2019 19:24:55 +0800 Subject: [PATCH 20/36] Add CI back Former-commit-id: 51725a23f47d8432c1ee8f4d15bd5a6994c6f38e --- ci/function/file_transfer.groovy | 10 + ci/jenkinsfile/cleanup_dev.groovy | 13 + ci/jenkinsfile/cleanup_staging.groovy | 13 + ci/jenkinsfile/cluster_cleanup_dev.groovy | 13 + ci/jenkinsfile/cluster_deploy2dev.groovy | 24 + ci/jenkinsfile/cluster_dev_test.groovy | 12 + ci/jenkinsfile/deploy2dev.groovy | 16 + ci/jenkinsfile/deploy2staging.groovy | 16 + ci/jenkinsfile/dev_test.groovy | 28 + ci/jenkinsfile/dev_test_all.groovy | 29 ++ ci/jenkinsfile/milvus_build.groovy | 30 ++ ci/jenkinsfile/milvus_build_no_ut.groovy | 29 ++ ci/jenkinsfile/nightly_publish_docker.groovy | 38 ++ ci/jenkinsfile/notify.groovy | 15 + ci/jenkinsfile/packaged_milvus.groovy | 44 ++ ci/jenkinsfile/packaged_milvus_no_ut.groovy | 26 + ci/jenkinsfile/publish_docker.groovy | 35 ++ ci/jenkinsfile/staging_test.groovy | 31 ++ .../upload_dev_cluster_test_out.groovy | 14 + ci/jenkinsfile/upload_dev_test_out.groovy | 13 + ci/jenkinsfile/upload_staging_test_out.groovy | 13 + ci/main_jenkinsfile | 383 ++++++++++++++ ci/main_jenkinsfile_no_ut | 383 ++++++++++++++ ci/nightly_main_jenkinsfile | 478 ++++++++++++++++++ ci/pod_containers/milvus-engine-build.yaml | 13 + ci/pod_containers/milvus-testframework.yaml | 13 + ci/pod_containers/publish-docker.yaml | 22 + 27 files changed, 1754 insertions(+) create mode 100644 ci/function/file_transfer.groovy create mode 100644 ci/jenkinsfile/cleanup_dev.groovy create mode 100644 ci/jenkinsfile/cleanup_staging.groovy create mode 100644 ci/jenkinsfile/cluster_cleanup_dev.groovy create mode 100644 ci/jenkinsfile/cluster_deploy2dev.groovy create mode 100644 ci/jenkinsfile/cluster_dev_test.groovy create mode 100644 ci/jenkinsfile/deploy2dev.groovy create mode 100644 ci/jenkinsfile/deploy2staging.groovy create mode 100644 ci/jenkinsfile/dev_test.groovy create mode 100644 ci/jenkinsfile/dev_test_all.groovy create mode 100644 ci/jenkinsfile/milvus_build.groovy create mode 100644 ci/jenkinsfile/milvus_build_no_ut.groovy create mode 100644 ci/jenkinsfile/nightly_publish_docker.groovy create mode 100644 ci/jenkinsfile/notify.groovy create mode 100644 ci/jenkinsfile/packaged_milvus.groovy create mode 100644 ci/jenkinsfile/packaged_milvus_no_ut.groovy create mode 100644 ci/jenkinsfile/publish_docker.groovy create mode 100644 ci/jenkinsfile/staging_test.groovy create mode 100644 ci/jenkinsfile/upload_dev_cluster_test_out.groovy create mode 100644 ci/jenkinsfile/upload_dev_test_out.groovy create mode 100644 ci/jenkinsfile/upload_staging_test_out.groovy create mode 100644 ci/main_jenkinsfile create mode 100644 ci/main_jenkinsfile_no_ut create mode 100644 ci/nightly_main_jenkinsfile create mode 100644 ci/pod_containers/milvus-engine-build.yaml create mode 100644 ci/pod_containers/milvus-testframework.yaml create mode 100644 ci/pod_containers/publish-docker.yaml diff --git a/ci/function/file_transfer.groovy b/ci/function/file_transfer.groovy new file mode 100644 index 0000000000..bebae14832 --- /dev/null +++ b/ci/function/file_transfer.groovy @@ -0,0 +1,10 @@ +def FileTransfer (sourceFiles, remoteDirectory, remoteIP, protocol = "ftp", makeEmptyDirs = true) { + if (protocol == "ftp") { + ftpPublisher masterNodeName: '', paramPublish: [parameterName: ''], alwaysPublishFromMaster: false, continueOnError: false, failOnError: true, publishers: [ + [configName: "${remoteIP}", transfers: [ + [asciiMode: false, cleanRemote: false, excludes: '', flatten: false, makeEmptyDirs: "${makeEmptyDirs}", noDefaultExcludes: false, patternSeparator: '[, ]+', remoteDirectory: "${remoteDirectory}", remoteDirectorySDF: false, removePrefix: '', sourceFiles: "${sourceFiles}"]], usePromotionTimestamp: true, useWorkspaceInPromotion: false, verbose: true + ] + ] + } +} +return this diff --git a/ci/jenkinsfile/cleanup_dev.groovy b/ci/jenkinsfile/cleanup_dev.groovy new file mode 100644 index 0000000000..2e9332fa6e --- /dev/null +++ b/ci/jenkinsfile/cleanup_dev.groovy @@ -0,0 +1,13 @@ +try { + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + } +} catch (exc) { + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + } + throw exc +} + diff --git a/ci/jenkinsfile/cleanup_staging.groovy b/ci/jenkinsfile/cleanup_staging.groovy new file mode 100644 index 0000000000..2e9332fa6e --- /dev/null +++ b/ci/jenkinsfile/cleanup_staging.groovy @@ -0,0 +1,13 @@ +try { + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + } +} catch (exc) { + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + } + throw exc +} + diff --git a/ci/jenkinsfile/cluster_cleanup_dev.groovy b/ci/jenkinsfile/cluster_cleanup_dev.groovy new file mode 100644 index 0000000000..e57988fefe --- /dev/null +++ b/ci/jenkinsfile/cluster_cleanup_dev.groovy @@ -0,0 +1,13 @@ +try { + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" + } +} catch (exc) { + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" + } + throw exc +} + diff --git a/ci/jenkinsfile/cluster_deploy2dev.groovy b/ci/jenkinsfile/cluster_deploy2dev.groovy new file mode 100644 index 0000000000..7f2a584256 --- /dev/null +++ b/ci/jenkinsfile/cluster_deploy2dev.groovy @@ -0,0 +1,24 @@ +try { + sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' + sh 'helm repo update' + dir ("milvus-helm") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + dir ("milvus/milvus-cluster") { + sh "helm install --wait --timeout 300 --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.5.0 . " + } + } + /* + timeout(time: 2, unit: 'MINUTES') { + waitUntil { + def result = sh script: "nc -z -w 3 ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local 19530", returnStatus: true + return !result + } + } + */ +} catch (exc) { + echo 'Helm running failed!' + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" + throw exc +} + diff --git a/ci/jenkinsfile/cluster_dev_test.groovy b/ci/jenkinsfile/cluster_dev_test.groovy new file mode 100644 index 0000000000..4a15b926cf --- /dev/null +++ b/ci/jenkinsfile/cluster_dev_test.groovy @@ -0,0 +1,12 @@ +timeout(time: 25, unit: 'MINUTES') { + try { + dir ("${PROJECT_NAME}_test") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + sh 'python3 -m pip install -r requirements_cluster.txt' + sh "pytest . --alluredir=cluster_test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local" + } + } catch (exc) { + echo 'Milvus Cluster Test Failed !' + throw exc + } +} diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy new file mode 100644 index 0000000000..b00c8fa335 --- /dev/null +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -0,0 +1,16 @@ +try { + sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' + sh 'helm repo update' + dir ("milvus-helm") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + dir ("milvus/milvus-gpu") { + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.5.0 ." + } + } +} catch (exc) { + echo 'Helm running failed!' + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + throw exc +} + diff --git a/ci/jenkinsfile/deploy2staging.groovy b/ci/jenkinsfile/deploy2staging.groovy new file mode 100644 index 0000000000..42ccfda71a --- /dev/null +++ b/ci/jenkinsfile/deploy2staging.groovy @@ -0,0 +1,16 @@ +try { + sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' + sh 'helm repo update' + dir ("milvus-helm") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + dir ("milvus/milvus-gpu") { + sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.5.0 ." + } + } +} catch (exc) { + echo 'Helm running failed!' + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + throw exc +} + diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy new file mode 100644 index 0000000000..f9df9b4065 --- /dev/null +++ b/ci/jenkinsfile/dev_test.groovy @@ -0,0 +1,28 @@ +timeout(time: 30, unit: 'MINUTES') { + try { + dir ("${PROJECT_NAME}_test") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local" + } + // mysql database backend test + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + + if (!fileExists('milvus-helm')) { + dir ("milvus-helm") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + } + } + dir ("milvus-helm") { + dir ("milvus/milvus-gpu") { + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.5.0 ." + } + } + dir ("${PROJECT_NAME}_test") { + sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local" + } + } catch (exc) { + echo 'Milvus Test Failed !' + throw exc + } +} diff --git a/ci/jenkinsfile/dev_test_all.groovy b/ci/jenkinsfile/dev_test_all.groovy new file mode 100644 index 0000000000..b11ea755b9 --- /dev/null +++ b/ci/jenkinsfile/dev_test_all.groovy @@ -0,0 +1,29 @@ +timeout(time: 60, unit: 'MINUTES') { + try { + dir ("${PROJECT_NAME}_test") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local" + } + + // mysql database backend test + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + + if (!fileExists('milvus-helm')) { + dir ("milvus-helm") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + } + } + dir ("milvus-helm") { + dir ("milvus/milvus-gpu") { + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ." + } + } + dir ("${PROJECT_NAME}_test") { + sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local" + } + } catch (exc) { + echo 'Milvus Test Failed !' + throw exc + } +} diff --git a/ci/jenkinsfile/milvus_build.groovy b/ci/jenkinsfile/milvus_build.groovy new file mode 100644 index 0000000000..7b0da2116c --- /dev/null +++ b/ci/jenkinsfile/milvus_build.groovy @@ -0,0 +1,30 @@ +container('milvus-build-env') { + timeout(time: 120, unit: 'MINUTES') { + gitlabCommitStatus(name: 'Build Engine') { + dir ("milvus_engine") { + try { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + + dir ("core") { + sh "git config --global user.email \"test@zilliz.com\"" + sh "git config --global user.name \"test\"" + withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + sh "./build.sh -l" + sh "rm -rf cmake_build" + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' \ + && export JFROG_USER_NAME='${USERNAME}' \ + && export JFROG_PASSWORD='${PASSWORD}' \ + && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.2.1/faiss-branch-0.2.1.tar.gz' \ + && ./build.sh -t ${params.BUILD_TYPE} -j -u -c" + + sh "./coverage.sh -u root -p 123456 -t 192.168.1.194" + } + } + } catch (exc) { + updateGitlabCommitStatus name: 'Build Engine', state: 'failed' + throw exc + } + } + } + } +} diff --git a/ci/jenkinsfile/milvus_build_no_ut.groovy b/ci/jenkinsfile/milvus_build_no_ut.groovy new file mode 100644 index 0000000000..d339f8c449 --- /dev/null +++ b/ci/jenkinsfile/milvus_build_no_ut.groovy @@ -0,0 +1,29 @@ +container('milvus-build-env') { + timeout(time: 120, unit: 'MINUTES') { + gitlabCommitStatus(name: 'Build Engine') { + dir ("milvus_engine") { + try { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + + dir ("core") { + sh "git config --global user.email \"test@zilliz.com\"" + sh "git config --global user.name \"test\"" + withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + sh "./build.sh -l" + sh "rm -rf cmake_build" + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' \ + && export JFROG_USER_NAME='${USERNAME}' \ + && export JFROG_PASSWORD='${PASSWORD}' \ + && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.2.1/faiss-branch-0.2.1.tar.gz' \ + && ./build.sh -t ${params.BUILD_TYPE} -j" + sh "./coverage.sh -u root -p 123456 -t 192.168.1.194" + } + } + } catch (exc) { + updateGitlabCommitStatus name: 'Build Engine', state: 'failed' + throw exc + } + } + } + } +} diff --git a/ci/jenkinsfile/nightly_publish_docker.groovy b/ci/jenkinsfile/nightly_publish_docker.groovy new file mode 100644 index 0000000000..8c6121bec8 --- /dev/null +++ b/ci/jenkinsfile/nightly_publish_docker.groovy @@ -0,0 +1,38 @@ +container('publish-docker') { + timeout(time: 15, unit: 'MINUTES') { + gitlabCommitStatus(name: 'Publish Engine Docker') { + try { + dir ("${PROJECT_NAME}_build") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:build/milvus_build.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + dir ("docker/deploy/ubuntu16.04/free_version") { + sh "curl -O -u anonymous: ftp://192.168.1.126/data/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" + sh "tar zxvf ${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" + try { + def customImage = docker.build("${PROJECT_NAME}/engine:${DOCKER_VERSION}") + docker.withRegistry('https://registry.zilliz.com', "${params.DOCKER_PUBLISH_USER}") { + customImage.push() + } + docker.withRegistry('https://zilliz.azurecr.cn', "${params.AZURE_DOCKER_PUBLISH_USER}") { + customImage.push() + } + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'success' + echo "Docker Pull Command: docker pull registry.zilliz.com/${PROJECT_NAME}/engine:${DOCKER_VERSION}" + } + } catch (exc) { + updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'canceled' + throw exc + } finally { + sh "docker rmi ${PROJECT_NAME}/engine:${DOCKER_VERSION}" + } + } + } + } catch (exc) { + updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'failed' + echo 'Publish docker failed!' + throw exc + } + } + } +} + diff --git a/ci/jenkinsfile/notify.groovy b/ci/jenkinsfile/notify.groovy new file mode 100644 index 0000000000..0a257b8cd8 --- /dev/null +++ b/ci/jenkinsfile/notify.groovy @@ -0,0 +1,15 @@ +def notify() { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } +} +return this + diff --git a/ci/jenkinsfile/packaged_milvus.groovy b/ci/jenkinsfile/packaged_milvus.groovy new file mode 100644 index 0000000000..1d30e21910 --- /dev/null +++ b/ci/jenkinsfile/packaged_milvus.groovy @@ -0,0 +1,44 @@ +container('milvus-build-env') { + timeout(time: 5, unit: 'MINUTES') { + dir ("milvus_engine") { + dir ("core") { + gitlabCommitStatus(name: 'Packaged Engine') { + if (fileExists('milvus')) { + try { + sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus" + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Download Milvus Engine Binary Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz\"" + } + } catch (exc) { + updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' + throw exc + } + } else { + updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' + error("Milvus binary directory don't exists!") + } + } + + gitlabCommitStatus(name: 'Packaged Engine lcov') { + if (fileExists('lcov_out')) { + try { + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("lcov_out/", "${PROJECT_NAME}/lcov/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Milvus lcov out Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/lcov/${JOB_NAME}-${BUILD_ID}/lcov_out/\"" + } + } catch (exc) { + updateGitlabCommitStatus name: 'Packaged Engine lcov', state: 'failed' + throw exc + } + } else { + updateGitlabCommitStatus name: 'Packaged Engine lcov', state: 'failed' + error("Milvus lcov out directory don't exists!") + } + } + } + } + } +} diff --git a/ci/jenkinsfile/packaged_milvus_no_ut.groovy b/ci/jenkinsfile/packaged_milvus_no_ut.groovy new file mode 100644 index 0000000000..bc68be374a --- /dev/null +++ b/ci/jenkinsfile/packaged_milvus_no_ut.groovy @@ -0,0 +1,26 @@ +container('milvus-build-env') { + timeout(time: 5, unit: 'MINUTES') { + dir ("milvus_engine") { + dir ("core") { + gitlabCommitStatus(name: 'Packaged Engine') { + if (fileExists('milvus')) { + try { + sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus" + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Download Milvus Engine Binary Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz\"" + } + } catch (exc) { + updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' + throw exc + } + } else { + updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' + error("Milvus binary directory don't exists!") + } + } + } + } + } +} diff --git a/ci/jenkinsfile/publish_docker.groovy b/ci/jenkinsfile/publish_docker.groovy new file mode 100644 index 0000000000..ef31eba9a4 --- /dev/null +++ b/ci/jenkinsfile/publish_docker.groovy @@ -0,0 +1,35 @@ +container('publish-docker') { + timeout(time: 15, unit: 'MINUTES') { + gitlabCommitStatus(name: 'Publish Engine Docker') { + try { + dir ("${PROJECT_NAME}_build") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:build/milvus_build.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + dir ("docker/deploy/ubuntu16.04/free_version") { + sh "curl -O -u anonymous: ftp://192.168.1.126/data/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" + sh "tar zxvf ${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" + try { + def customImage = docker.build("${PROJECT_NAME}/engine:${DOCKER_VERSION}") + docker.withRegistry('https://registry.zilliz.com', "${params.DOCKER_PUBLISH_USER}") { + customImage.push() + } + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'success' + echo "Docker Pull Command: docker pull registry.zilliz.com/${PROJECT_NAME}/engine:${DOCKER_VERSION}" + } + } catch (exc) { + updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'canceled' + throw exc + } finally { + sh "docker rmi ${PROJECT_NAME}/engine:${DOCKER_VERSION}" + } + } + } + } catch (exc) { + updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'failed' + echo 'Publish docker failed!' + throw exc + } + } + } +} + diff --git a/ci/jenkinsfile/staging_test.groovy b/ci/jenkinsfile/staging_test.groovy new file mode 100644 index 0000000000..dcf1787103 --- /dev/null +++ b/ci/jenkinsfile/staging_test.groovy @@ -0,0 +1,31 @@ +timeout(time: 40, unit: 'MINUTES') { + try { + dir ("${PROJECT_NAME}_test") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + sh 'python3 -m pip install -r requirements.txt' + def service_ip = sh (script: "kubectl get svc --namespace milvus-1 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim() + sh "pytest . --alluredir=\"test_out/staging/single/sqlite\" --ip ${service_ip}" + } + + // mysql database backend test + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy" + + if (!fileExists('milvus-helm')) { + dir ("milvus-helm") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + } + } + dir ("milvus-helm") { + dir ("milvus/milvus-gpu") { + sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.5.0 ." + } + } + dir ("${PROJECT_NAME}_test") { + def service_ip = sh (script: "kubectl get svc --namespace milvus-2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim() + sh "pytest . --alluredir=\"test_out/staging/single/mysql\" --ip ${service_ip}" + } + } catch (exc) { + echo 'Milvus Test Failed !' + throw exc + } +} diff --git a/ci/jenkinsfile/upload_dev_cluster_test_out.groovy b/ci/jenkinsfile/upload_dev_cluster_test_out.groovy new file mode 100644 index 0000000000..6bbd8a649f --- /dev/null +++ b/ci/jenkinsfile/upload_dev_cluster_test_out.groovy @@ -0,0 +1,14 @@ +timeout(time: 5, unit: 'MINUTES') { + dir ("${PROJECT_NAME}_test") { + if (fileExists('cluster_test_out')) { + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("cluster_test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" + } + } else { + error("Milvus Dev Test Out directory don't exists!") + } + } +} + diff --git a/ci/jenkinsfile/upload_dev_test_out.groovy b/ci/jenkinsfile/upload_dev_test_out.groovy new file mode 100644 index 0000000000..017b887334 --- /dev/null +++ b/ci/jenkinsfile/upload_dev_test_out.groovy @@ -0,0 +1,13 @@ +timeout(time: 5, unit: 'MINUTES') { + dir ("${PROJECT_NAME}_test") { + if (fileExists('test_out/dev')) { + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("test_out/dev/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" + } + } else { + error("Milvus Dev Test Out directory don't exists!") + } + } +} diff --git a/ci/jenkinsfile/upload_staging_test_out.groovy b/ci/jenkinsfile/upload_staging_test_out.groovy new file mode 100644 index 0000000000..1f1e66ab1b --- /dev/null +++ b/ci/jenkinsfile/upload_staging_test_out.groovy @@ -0,0 +1,13 @@ +timeout(time: 5, unit: 'MINUTES') { + dir ("${PROJECT_NAME}_test") { + if (fileExists('test_out/staging')) { + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("test_out/staging/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" + } + } else { + error("Milvus Dev Test Out directory don't exists!") + } + } +} diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile new file mode 100644 index 0000000000..12c6c81cfd --- /dev/null +++ b/ci/main_jenkinsfile @@ -0,0 +1,383 @@ +pipeline { + agent none + + options { + timestamps() + } + + environment { + PROJECT_NAME = "milvus" + LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() + SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" + GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" + SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" + DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}" + } + + stages { + stage("Ubuntu 16.04") { + environment { + PACKAGE_VERSION = VersionNumber([ + versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}' + ]); + + DOCKER_VERSION = VersionNumber([ + versionNumberString : '${DOCKER_VERSION_STR}' + ]); + } + + stages { + stage("Run Build") { + agent { + kubernetes { + cloud 'build-kubernetes' + label 'build' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + name: milvus-build-env + labels: + app: milvus + componet: build-env +spec: + containers: + - name: milvus-build-env + image: registry.zilliz.com/milvus/milvus-build-env:v0.13 + command: + - cat + tty: true + resources: + limits: + memory: "28Gi" + cpu: "10.0" + nvidia.com/gpu: 1 + requests: + memory: "14Gi" + cpu: "5.0" +""" + } + } + stages { + stage('Build') { + steps { + gitlabCommitStatus(name: 'Build') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Build', state: 'canceled' + echo "Milvus Build aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Build', state: 'failed' + echo "Milvus Build failure !" + } + } + } + } + + stage("Publish docker and helm") { + agent { + kubernetes { + label 'publish' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: publish + componet: docker +spec: + containers: + - name: publish-docker + image: registry.zilliz.com/library/zilliz_docker:v1.0.0 + securityContext: + privileged: true + command: + - cat + tty: true + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock +""" + } + } + stages { + stage('Publish Docker') { + steps { + gitlabCommitStatus(name: 'Publish Docker') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled' + echo "Milvus Publish Docker aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Publish Docker', state: 'failed' + echo "Milvus Publish Docker failure !" + } + } + } + } + + stage("Deploy to Development") { + parallel { + stage("Single Node") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } + + stages { + stage("Deploy to Dev") { + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" + } + } + } + } + } + stage("Dev Test") { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" + } + } + } + } + } + stage ("Cleanup Dev") { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + success { + script { + echo "Milvus Single Node CI/CD success !" + } + } + aborted { + script { + echo "Milvus Single Node CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Single Node CI/CD failure !" + } + } + } + } + +// stage("Cluster") { +// agent { +// kubernetes { +// label 'dev-test' +// defaultContainer 'jnlp' +// yaml """ +// apiVersion: v1 +// kind: Pod +// metadata: +// labels: +// app: milvus +// componet: test +// spec: +// containers: +// - name: milvus-testframework +// image: registry.zilliz.com/milvus/milvus-test:v0.2 +// command: +// - cat +// tty: true +// volumeMounts: +// - name: kubeconf +// mountPath: /root/.kube/ +// readOnly: true +// volumes: +// - name: kubeconf +// secret: +// secretName: test-cluster-config +// """ +// } +// } +// stages { +// stage("Deploy to Dev") { +// steps { +// gitlabCommitStatus(name: 'Deloy to Dev') { +// container('milvus-testframework') { +// script { +// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" +// } +// } +// } +// } +// } +// stage("Dev Test") { +// steps { +// gitlabCommitStatus(name: 'Deloy Test') { +// container('milvus-testframework') { +// script { +// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" +// load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" +// } +// } +// } +// } +// } +// stage ("Cleanup Dev") { +// steps { +// gitlabCommitStatus(name: 'Cleanup Dev') { +// container('milvus-testframework') { +// script { +// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" +// } +// } +// } +// } +// } +// } +// post { +// always { +// container('milvus-testframework') { +// script { +// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" +// } +// } +// } +// success { +// script { +// echo "Milvus Cluster CI/CD success !" +// } +// } +// aborted { +// script { +// echo "Milvus Cluster CI/CD aborted !" +// } +// } +// failure { +// script { +// echo "Milvus Cluster CI/CD failure !" +// } +// } +// } +// } + } + } + } + } + } + + post { + always { + script { + if (env.gitlabAfter != null) { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } + } + } + } + + success { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'success' + echo "Milvus CI/CD success !" + } + } + + aborted { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'canceled' + echo "Milvus CI/CD aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'failed' + echo "Milvus CI/CD failure !" + } + } + } +} + diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut new file mode 100644 index 0000000000..e7382bd1fd --- /dev/null +++ b/ci/main_jenkinsfile_no_ut @@ -0,0 +1,383 @@ +pipeline { + agent none + + options { + timestamps() + } + + environment { + PROJECT_NAME = "milvus" + LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() + SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" + GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" + SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" + DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}" + } + + stages { + stage("Ubuntu 16.04") { + environment { + PACKAGE_VERSION = VersionNumber([ + versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}' + ]); + + DOCKER_VERSION = VersionNumber([ + versionNumberString : '${DOCKER_VERSION_STR}' + ]); + } + + stages { + stage("Run Build") { + agent { + kubernetes { + cloud 'build-kubernetes' + label 'build' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + name: milvus-build-env + labels: + app: milvus + componet: build-env +spec: + containers: + - name: milvus-build-env + image: registry.zilliz.com/milvus/milvus-build-env:v0.13 + command: + - cat + tty: true + resources: + limits: + memory: "28Gi" + cpu: "10.0" + nvidia.com/gpu: 1 + requests: + memory: "14Gi" + cpu: "5.0" +""" + } + } + stages { + stage('Build') { + steps { + gitlabCommitStatus(name: 'Build') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build_no_ut.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus_no_ut.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Build', state: 'canceled' + echo "Milvus Build aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Build', state: 'failed' + echo "Milvus Build failure !" + } + } + } + } + + stage("Publish docker and helm") { + agent { + kubernetes { + label 'publish' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: publish + componet: docker +spec: + containers: + - name: publish-docker + image: registry.zilliz.com/library/zilliz_docker:v1.0.0 + securityContext: + privileged: true + command: + - cat + tty: true + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock +""" + } + } + stages { + stage('Publish Docker') { + steps { + gitlabCommitStatus(name: 'Publish Docker') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled' + echo "Milvus Publish Docker aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Publish Docker', state: 'failed' + echo "Milvus Publish Docker failure !" + } + } + } + } + + stage("Deploy to Development") { + parallel { + stage("Single Node") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } + + stages { + stage("Deploy to Dev") { + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" + } + } + } + } + } + stage("Dev Test") { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" + } + } + } + } + } + stage ("Cleanup Dev") { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + success { + script { + echo "Milvus Single Node CI/CD success !" + } + } + aborted { + script { + echo "Milvus Single Node CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Single Node CI/CD failure !" + } + } + } + } + +// stage("Cluster") { +// agent { +// kubernetes { +// label 'dev-test' +// defaultContainer 'jnlp' +// yaml """ +// apiVersion: v1 +// kind: Pod +// metadata: +// labels: +// app: milvus +// componet: test +// spec: +// containers: +// - name: milvus-testframework +// image: registry.zilliz.com/milvus/milvus-test:v0.2 +// command: +// - cat +// tty: true +// volumeMounts: +// - name: kubeconf +// mountPath: /root/.kube/ +// readOnly: true +// volumes: +// - name: kubeconf +// secret: +// secretName: test-cluster-config +// """ +// } +// } +// stages { +// stage("Deploy to Dev") { +// steps { +// gitlabCommitStatus(name: 'Deloy to Dev') { +// container('milvus-testframework') { +// script { +// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" +// } +// } +// } +// } +// } +// stage("Dev Test") { +// steps { +// gitlabCommitStatus(name: 'Deloy Test') { +// container('milvus-testframework') { +// script { +// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" +// load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" +// } +// } +// } +// } +// } +// stage ("Cleanup Dev") { +// steps { +// gitlabCommitStatus(name: 'Cleanup Dev') { +// container('milvus-testframework') { +// script { +// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" +// } +// } +// } +// } +// } +// } +// post { +// always { +// container('milvus-testframework') { +// script { +// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" +// } +// } +// } +// success { +// script { +// echo "Milvus Cluster CI/CD success !" +// } +// } +// aborted { +// script { +// echo "Milvus Cluster CI/CD aborted !" +// } +// } +// failure { +// script { +// echo "Milvus Cluster CI/CD failure !" +// } +// } +// } +// } + } + } + } + } + } + + post { + always { + script { + if (env.gitlabAfter != null) { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } + } + } + } + + success { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'success' + echo "Milvus CI/CD success !" + } + } + + aborted { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'canceled' + echo "Milvus CI/CD aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'failed' + echo "Milvus CI/CD failure !" + } + } + } +} + diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile new file mode 100644 index 0000000000..add9e00fb4 --- /dev/null +++ b/ci/nightly_main_jenkinsfile @@ -0,0 +1,478 @@ +pipeline { + agent none + + options { + timestamps() + } + + environment { + PROJECT_NAME = "milvus" + LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() + SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" + GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" + SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" + DOCKER_VERSION_STR = "${env.gitlabAfter == null ? '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, \"yyyyMMdd\"}' : '${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}'}" + } + + stages { + stage("Ubuntu 16.04") { + environment { + PACKAGE_VERSION = VersionNumber([ + versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}' + ]); + + DOCKER_VERSION = VersionNumber([ + versionNumberString : '${DOCKER_VERSION_STR}' + ]); + } + + stages { + stage("Run Build") { + agent { + kubernetes { + cloud 'build-kubernetes' + label 'build' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + name: milvus-build-env + labels: + app: milvus + componet: build-env +spec: + containers: + - name: milvus-build-env + image: registry.zilliz.com/milvus/milvus-build-env:v0.13 + command: + - cat + tty: true + resources: + limits: + memory: "28Gi" + cpu: "10.0" + nvidia.com/gpu: 1 + requests: + memory: "14Gi" + cpu: "5.0" +""" + } + } + stages { + stage('Build') { + steps { + gitlabCommitStatus(name: 'Build') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Build', state: 'canceled' + echo "Milvus Build aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Build', state: 'failed' + echo "Milvus Build failure !" + } + } + } + } + + stage("Publish docker and helm") { + agent { + kubernetes { + label 'publish' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: publish + componet: docker +spec: + containers: + - name: publish-docker + image: registry.zilliz.com/library/zilliz_docker:v1.0.0 + securityContext: + privileged: true + command: + - cat + tty: true + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock +""" + } + } + stages { + stage('Publish Docker') { + steps { + gitlabCommitStatus(name: 'Publish Docker') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/nightly_publish_docker.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled' + echo "Milvus Publish Docker aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Publish Docker', state: 'failed' + echo "Milvus Publish Docker failure !" + } + } + } + } + + stage("Deploy to Development") { + parallel { + stage("Single Node") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } + + stages { + stage("Deploy to Dev") { + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" + } + } + } + } + } + stage("Dev Test") { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/dev_test_all.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" + } + } + } + } + } + stage ("Cleanup Dev") { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + success { + script { + echo "Milvus Deploy to Dev Single Node CI/CD success !" + } + } + aborted { + script { + echo "Milvus Deploy to Dev Single Node CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Deploy to Dev Single Node CI/CD failure !" + } + } + } + } + +// stage("Cluster") { +// agent { +// kubernetes { +// label 'dev-test' +// defaultContainer 'jnlp' +// yaml """ +// apiVersion: v1 +// kind: Pod +// metadata: +// labels: +// app: milvus +// componet: test +// spec: +// containers: +// - name: milvus-testframework +// image: registry.zilliz.com/milvus/milvus-test:v0.2 +// command: +// - cat +// tty: true +// volumeMounts: +// - name: kubeconf +// mountPath: /root/.kube/ +// readOnly: true +// volumes: +// - name: kubeconf +// secret: +// secretName: test-cluster-config +// """ +// } +// } +// stages { +// stage("Deploy to Dev") { +// steps { +// gitlabCommitStatus(name: 'Deloy to Dev') { +// container('milvus-testframework') { +// script { +// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" +// } +// } +// } +// } +// } +// stage("Dev Test") { +// steps { +// gitlabCommitStatus(name: 'Deloy Test') { +// container('milvus-testframework') { +// script { +// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" +// load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" +// } +// } +// } +// } +// } +// stage ("Cleanup Dev") { +// steps { +// gitlabCommitStatus(name: 'Cleanup Dev') { +// container('milvus-testframework') { +// script { +// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" +// } +// } +// } +// } +// } +// } +// post { +// always { +// container('milvus-testframework') { +// script { +// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" +// } +// } +// } +// success { +// script { +// echo "Milvus Deploy to Dev Cluster CI/CD success !" +// } +// } +// aborted { +// script { +// echo "Milvus Deploy to Dev Cluster CI/CD aborted !" +// } +// } +// failure { +// script { +// echo "Milvus Deploy to Dev Cluster CI/CD failure !" +// } +// } +// } +// } + } + } + + stage("Deploy to Staging") { + parallel { + stage("Single Node") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: aks-gpu-cluster-config +""" + } + } + + stages { + stage("Deploy to Staging") { + steps { + gitlabCommitStatus(name: 'Deloy to Staging') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2staging.groovy" + } + } + } + } + } + stage("Staging Test") { + steps { + gitlabCommitStatus(name: 'Staging Test') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/staging_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_staging_test_out.groovy" + } + } + } + } + } + stage ("Cleanup Staging") { + steps { + gitlabCommitStatus(name: 'Cleanup Staging') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy" + } + } + } + } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy" + } + } + } + success { + script { + echo "Milvus Deploy to Staging Single Node CI/CD success !" + } + } + aborted { + script { + echo "Milvus Deploy to Staging Single Node CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Deploy to Staging Single Node CI/CD failure !" + } + } + } + } + } + } + } + } + } + + post { + always { + script { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } + } + } + + success { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'success' + echo "Milvus CI/CD success !" + } + } + + aborted { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'canceled' + echo "Milvus CI/CD aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'failed' + echo "Milvus CI/CD failure !" + } + } + } +} + diff --git a/ci/pod_containers/milvus-engine-build.yaml b/ci/pod_containers/milvus-engine-build.yaml new file mode 100644 index 0000000000..cd5352ffef --- /dev/null +++ b/ci/pod_containers/milvus-engine-build.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: build-env +spec: + containers: + - name: milvus-build-env + image: registry.zilliz.com/milvus/milvus-build-env:v0.9 + command: + - cat + tty: true diff --git a/ci/pod_containers/milvus-testframework.yaml b/ci/pod_containers/milvus-testframework.yaml new file mode 100644 index 0000000000..7a98fbca8e --- /dev/null +++ b/ci/pod_containers/milvus-testframework.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: testframework +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.1 + command: + - cat + tty: true diff --git a/ci/pod_containers/publish-docker.yaml b/ci/pod_containers/publish-docker.yaml new file mode 100644 index 0000000000..268afb1331 --- /dev/null +++ b/ci/pod_containers/publish-docker.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: publish + componet: docker +spec: + containers: + - name: publish-docker + image: registry.zilliz.com/library/zilliz_docker:v1.0.0 + securityContext: + privileged: true + command: + - cat + tty: true + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock From 4b3f1f056c80931185176e31b11ed5cc1ccce4f8 Mon Sep 17 00:00:00 2001 From: starlord Date: Tue, 15 Oct 2019 20:46:34 +0800 Subject: [PATCH 21/36] remove zilliz Former-commit-id: 1470a09bbb1c0f14c63c7bd9f718f9b65b82d88b --- core/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main.cpp b/core/src/main.cpp index 4178626f32..d1c9ba6dfd 100644 --- a/core/src/main.cpp +++ b/core/src/main.cpp @@ -50,7 +50,7 @@ print_banner() { std::cout << " / /|_/ // // /_| |/ / /_/ /\\ \\ " << std::endl; std::cout << " /_/ /_/___/____/___/\\____/___/ " << std::endl; std::cout << std::endl; - std::cout << "Welcome to use Milvus!" << std::endl; + std::cout << "Welcome to Milvus!" << std::endl; std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME << std::endl; std::cout << std::endl; } From 839009ff26c65db7c014465f01c039a708a4132a Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Tue, 15 Oct 2019 20:51:30 +0800 Subject: [PATCH 22/36] Use system boost anymore Former-commit-id: 159bcdb3599fb72ddbd9b35b388ced78150deb03 --- core/cmake/DefineOptions.cmake | 2 +- core/src/CMakeLists.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/cmake/DefineOptions.cmake b/core/cmake/DefineOptions.cmake index 1b0646c2fa..7aae177f0b 100644 --- a/core/cmake/DefineOptions.cmake +++ b/core/cmake/DefineOptions.cmake @@ -56,7 +56,7 @@ define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD "Show output from ExternalProjects rather than just logging to files" ON) define_option(MILVUS_BOOST_VENDORED "Use vendored Boost instead of existing Boost. \ -Note that this requires linking Boost statically" ON) +Note that this requires linking Boost statically" OFF) define_option(MILVUS_BOOST_HEADER_ONLY "Use only BOOST headers" OFF) diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index 5f87d1d219..499800c655 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -96,9 +96,9 @@ set(prometheus_lib ) set(boost_lib - boost_system_static - boost_filesystem_static - boost_serialization_static + boost_system + boost_filesystem + boost_serialization ) set(cuda_lib From b15f0e36a5d4cdc9809c8f19417a0cb19fe5adae Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Tue, 15 Oct 2019 20:53:02 +0800 Subject: [PATCH 23/36] Update ubuntu build depends Former-commit-id: 2c1c0c8286d5c14c1c9d6542a489ab82cf344d01 --- core/ubuntu_build_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/ubuntu_build_deps.sh b/core/ubuntu_build_deps.sh index 06f05fa49f..ed9eb9dee5 100755 --- a/core/ubuntu_build_deps.sh +++ b/core/ubuntu_build_deps.sh @@ -1,5 +1,5 @@ #!/bin/bash -sudo apt-get install -y gfortran libmysqlclient-dev mysql-client libcurl4-openssl-dev +sudo apt-get install -y gfortran libmysqlclient-dev mysql-client libcurl4-openssl-dev libboost-system-dev libboost-filesystem-dev libboost-serialization-dev sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so From ae30b7fbe589a2e56c11257c5d17ab0ab0f69c83 Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Wed, 16 Oct 2019 10:15:34 +0800 Subject: [PATCH 24/36] #6 Use system boost instead of compiling boost source code. Former-commit-id: 982ca5d9cf921b325f6f92d7e52e08c8ee61c998 --- core/src/CMakeLists.txt | 6 +++--- core/unittest/CMakeLists.txt | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index 499800c655..b0228bd090 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -96,9 +96,9 @@ set(prometheus_lib ) set(boost_lib - boost_system - boost_filesystem - boost_serialization + libboost_system.a + libboost_filesystem.a + libboost_serialization.a ) set(cuda_lib diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index fe40e76afa..258fd76a8e 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -92,8 +92,8 @@ set(common_files set(unittest_libs sqlite - boost_system_static - boost_filesystem_static + libboost_system.a + libboost_filesystem.a lz4 mysqlpp yaml-cpp From 03adffe26fe3f638c6c0d94e7220adc9d8d3ee50 Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Wed, 16 Oct 2019 10:29:16 +0800 Subject: [PATCH 25/36] [BUG] Change milvus_server docker image to latest in docker-compose-monitor.yml Former-commit-id: be24b770839a3536a83ead9615ce16c1abbc3488 --- docker/docker-compose-monitor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/docker-compose-monitor.yml b/docker/docker-compose-monitor.yml index 81e7084af9..ac417674be 100644 --- a/docker/docker-compose-monitor.yml +++ b/docker/docker-compose-monitor.yml @@ -42,15 +42,15 @@ services: milvus_server: runtime: nvidia - image: registry.zilliz.com/milvus/engine:branch-0.4.0-release + image: milvusdb/milvus:latest restart: always links: - prometheus environment: WEB_APP: host.docker.internal volumes: - - ../cpp/conf/server_config.yaml:/opt/milvus/conf/server_config.yaml - - ../cpp/conf/log_config.conf:/opt/milvus/conf/log_config.conf + - ../core/conf/server_config.yaml:/opt/milvus/conf/server_config.yaml + - ../core/conf/log_config.conf:/opt/milvus/conf/log_config.conf ports: - "8080:8080" - "19530:19530" From e4e7dd7faad3e4e69da7229be615c86bcde93f2c Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 16 Oct 2019 11:55:36 +0800 Subject: [PATCH 26/36] fix jenkins build error Former-commit-id: 7e4db47a744e810698aff5377db52d85136dbb5b --- ci/jenkinsfile/milvus_build_no_ut.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/jenkinsfile/milvus_build_no_ut.groovy b/ci/jenkinsfile/milvus_build_no_ut.groovy index d339f8c449..24f2cedef4 100644 --- a/ci/jenkinsfile/milvus_build_no_ut.groovy +++ b/ci/jenkinsfile/milvus_build_no_ut.groovy @@ -16,7 +16,6 @@ container('milvus-build-env') { && export JFROG_PASSWORD='${PASSWORD}' \ && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.2.1/faiss-branch-0.2.1.tar.gz' \ && ./build.sh -t ${params.BUILD_TYPE} -j" - sh "./coverage.sh -u root -p 123456 -t 192.168.1.194" } } } catch (exc) { From e524bfb35c4ef9249003f83c6dbb77ae38562039 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 16 Oct 2019 11:57:11 +0800 Subject: [PATCH 27/36] change default gpu_cache_capacity to 4 Former-commit-id: d5a629cb39c58cc82696b6656fe67a1e3270a300 --- core/conf/server_config.template | 2 ++ core/src/server/Config.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/core/conf/server_config.template b/core/conf/server_config.template index 2f2f699e09..27c2b12cfe 100644 --- a/core/conf/server_config.template +++ b/core/conf/server_config.template @@ -30,6 +30,8 @@ metric_config: cache_config: cpu_cache_capacity: 16 # GB, CPU memory used for cache cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered + gpu_cache_capacity: 4 # GB, GPU memory used for cache + gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered cache_insert_data: false # whether to load inserted data into cache engine_config: diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 9c754256a2..f40c9f2224 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -62,7 +62,7 @@ static const char* CONFIG_CACHE = "cache_config"; static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY = "cpu_cache_capacity"; static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT = "16"; static const char* CONFIG_CACHE_GPU_CACHE_CAPACITY = "gpu_cache_capacity"; -static const char* CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT = "0"; +static const char* CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT = "4"; static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_mem_threshold"; static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT = "0.85"; static const char* CONFIG_CACHE_GPU_CACHE_THRESHOLD = "gpu_mem_threshold"; From af956f337fc3a629ba8d620ec7fcd0e3cc11aa9b Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 16 Oct 2019 14:25:24 +0800 Subject: [PATCH 28/36] Update README.md Former-commit-id: 69949ae305efa2033872e35f66b37e0a096dbe1e --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d2622816b7..5eec7f34cf 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ![Language](https://img.shields.io/badge/language-C%2B%2B-blue) - [Slack Community](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk) -- [Twitter](https://twitter.com/milvus_io) +- [Twitter](https://twitter.com/milvusio) - [Blog](https://www.milvus.io/blog/) - [CSDN](https://zilliz.blog.csdn.net/) - [中文官网](https://www.milvus.io/zh-CN/) From 7d8f7ac073a45788baa8351701bdb3b09f9f580a Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 16 Oct 2019 14:27:51 +0800 Subject: [PATCH 29/36] Update README.md Former-commit-id: 2b5651fa86b09ac5200b14218c9ee567a8a97405 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5eec7f34cf..0f24955a59 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![Milvuslogo](https://milvus.io/docs/assets/milvus_logo.png) +![Milvuslogo](https://milvus.io/docs/tree/branch-0.5.0/assets/milvus_logo.png) ![LICENSE](https://img.shields.io/badge/license-Apache--2.0-brightgreen) ![Language](https://img.shields.io/badge/language-C%2B%2B-blue) @@ -37,7 +37,7 @@ Keep up-to-date with newest releases and latest updates by reading Milvus [relea The data is stored and computed on a distributed architecture. This lets you scale data sizes up and down without redesigning the system. ## Architecture -![Milvus_arch](https://milvus.io/docs/assets/milvus_arch.png) +![Milvus_arch](https://milvus.io/docs/tree/branch-0.5.0/assets/milvus_arch.png) ## Get started From 24d3b14ba4741ef04e877c68772d77fef91b27f1 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 16 Oct 2019 14:29:25 +0800 Subject: [PATCH 30/36] Update README.md Former-commit-id: 1ed79b12556c70cb50db8c7ec36ec303cbd64d6a --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0f24955a59..5eec7f34cf 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![Milvuslogo](https://milvus.io/docs/tree/branch-0.5.0/assets/milvus_logo.png) +![Milvuslogo](https://milvus.io/docs/assets/milvus_logo.png) ![LICENSE](https://img.shields.io/badge/license-Apache--2.0-brightgreen) ![Language](https://img.shields.io/badge/language-C%2B%2B-blue) @@ -37,7 +37,7 @@ Keep up-to-date with newest releases and latest updates by reading Milvus [relea The data is stored and computed on a distributed architecture. This lets you scale data sizes up and down without redesigning the system. ## Architecture -![Milvus_arch](https://milvus.io/docs/tree/branch-0.5.0/assets/milvus_arch.png) +![Milvus_arch](https://milvus.io/docs/assets/milvus_arch.png) ## Get started From 75cc77e66a15935ec89f063d2f31d60cb9128166 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 16 Oct 2019 15:08:11 +0800 Subject: [PATCH 31/36] Update README.md Former-commit-id: 9c15b47dcfc3e3d9ea60b86985127aba39b2f9fb --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 5eec7f34cf..aa6d3fbb70 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,6 @@ Firstly, welcome, and thanks for your interest in [Milvus](https://milvus.io)! N Milvus is an open source vector search engine that supports similarity search of large-scale vectors. Built on optimized indexing algorithm, it is compatible with major AI/ML models. -Milvus was developed by ZILLIZ, a tech startup that intends to reinvent data science, with the purpose of providing enterprises with efficient and scalable similarity search and analysis of feature vectors and unstructured data. - Milvus provides stable Python, C++ and Java APIs. Keep up-to-date with newest releases and latest updates by reading Milvus [release notes](https://milvus.io/docs/en/Releases/v0.4.0/). From 9f138078c006fde9cb532462b6b3ebb1081b20c8 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 16 Oct 2019 15:27:10 +0800 Subject: [PATCH 32/36] Update README.md Former-commit-id: 212c3fe64bebab057feb08261529b952357aec56 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index aa6d3fbb70..ce3cd77f37 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![Milvuslogo](https://milvus.io/docs/assets/milvus_logo.png) +![Milvuslogo](https://github.com/milvus-io/docs/blob/branch-0.5.0/assets/milvus_logo.png) ![LICENSE](https://img.shields.io/badge/license-Apache--2.0-brightgreen) ![Language](https://img.shields.io/badge/language-C%2B%2B-blue) @@ -35,7 +35,7 @@ Keep up-to-date with newest releases and latest updates by reading Milvus [relea The data is stored and computed on a distributed architecture. This lets you scale data sizes up and down without redesigning the system. ## Architecture -![Milvus_arch](https://milvus.io/docs/assets/milvus_arch.png) +![Milvus_arch](https://github.com/milvus-io/docs/blob/branch-0.5.0/assets/milvus_arch.png) ## Get started From cb088c77782414a9b3a04b670da51628e3e20274 Mon Sep 17 00:00:00 2001 From: jielinxu Date: Wed, 16 Oct 2019 16:17:04 +0800 Subject: [PATCH 33/36] update README and CONTRIBUTING Former-commit-id: 4bf191eebdaa945ce58c5c31124f4837862960fe --- CONTRIBUTING.md | 56 ++++++++++++++++++++++++++++++++++++++++--------- README.md | 53 ++-------------------------------------------- 2 files changed, 48 insertions(+), 61 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 11068ca7e1..475387c319 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing to Milvus -First of all, thanks for taking the time to contribute to Milvus! It's people like you that help Milvus come to fruition. +First of all, thanks for taking the time to contribute to Milvus! It's people like you that help Milvus come to fruition. :tada: The following are a set of guidelines for contributing to Milvus. Following these guidelines helps contributing to this project easy and transparent. These are mostly guideline, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request. @@ -11,8 +11,8 @@ As for everything else in the project, the contributions to Milvus are governed Before you make any contributions, make sure you follow this list. - Read [Contributing to Milvus](CONTRIBUTING.md). -- Check if the changes are consistent with the [coding style](CONTRIBUTING.md#coding-style). -- Run [unit tests](CONTRIBUTING.md#run-unit-test). +- Check if the changes are consistent with the [coding style](CONTRIBUTING.md#coding-style), and format your code accordingly. +- Run [unit tests](CONTRIBUTING.md#run-unit-test-with-code-coverage) and check your code coverage rate. ## What contributions can I make? @@ -68,15 +68,51 @@ And we made the following changes based on the guide: - 120-character line length - Camel-Cased file names +### Format code -## Run unit test - -We use Google Test framework for test running. -To run unit test for Milvus under C++, please use the following command: - +Install clang-format ```shell -# Run unit test for Milvus -$ ./build.sh -u +$ sudo apt-get install clang-format +$ rm cmake_build/CMakeCache.txt +``` +Check code style +```shell +$ ./build.sh -l +``` +To format the code +```shell +$ cd cmake_build +$ make clang-format ``` +## Run unit test with code coverage + +Before submitting your PR, make sure you have run unit test, and your code coverage rate is >= 90%. + +Install lcov +```shell +$ sudo apt-get install lcov +``` +Run unit test and generate code for code coverage check +```shell +$ ./build.sh -u -c +``` + +Run MySQL docker +```shell +docker pull mysql:latest +docker run -p 3306:3306 -e MYSQL_ROOT_PASSWORD=123456 -d mysql:latest +``` + +Run code coverage + +```shell +$ ./coverage.sh -u root -p 123456 -t 127.0.0.1 +``` + +Or start your own MySQL server, and then run code coverage + +```shell +$ ./coverage.sh -u ${MYSQL_USERNAME} -p ${MYSQL_PASSWORD} -t ${MYSQL_SERVER_IP} +``` diff --git a/README.md b/README.md index ce3cd77f37..5ea44c3122 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ # Welcome to Milvus -Firstly, welcome, and thanks for your interest in [Milvus](https://milvus.io)! No matter who you are, what you do, we greatly appreciate your contribution to help us reinvent data science with Milvus. +Firstly, welcome, and thanks for your interest in [Milvus](https://milvus.io)! ​​No matter who you are, what you do, we greatly appreciate your contribution to help us reinvent data science with Milvus. ## What is Milvus @@ -28,7 +28,7 @@ Keep up-to-date with newest releases and latest updates by reading Milvus [relea - Intelligent index - With a “Decide Your Own Algorithm” approach, you can embed machine learning and advanced algorithms into Milvus without the headache of complex data engineering or migrating data between disparate systems. Milvus is built on optimized indexing algorithm based on quantization indexing, tree-based and graph indexing methods. + With a "Decide Your Own Algorithm" approach, you can embed machine learning and advanced algorithms into Milvus without the headache of complex data engineering or migrating data between disparate systems. Milvus is built on optimized indexing algorithm based on quantization indexing, tree-based and graph indexing methods. - Strong scalability @@ -72,55 +72,6 @@ $ ./build.sh -t Release When the build is completed, all the stuff that you need in order to run Milvus will be installed under `[Milvus root path]/core/milvus`. -#### Code format and linting - -Install clang-format -```shell -$ sudo apt-get install clang-format -$ rm cmake_build/CMakeCache.txt -``` -Check code style -```shell -$ ./build.sh -l -``` -To format the code -```shell -$ cd cmake_build -$ make clang-format -``` - -#### Run unit test - -```shell -$ ./build.sh -u -``` - -#### Run code coverage - -Install lcov -```shell -$ sudo apt-get install lcov -``` -```shell -$ ./build.sh -u -c -``` -Run MySQL docker -```shell -docker pull mysql:latest -docker run -p 3306:3306 -e MYSQL_ROOT_PASSWORD=123456 -d mysql:latest -``` -Run code coverage - -```shell -$ ./coverage.sh -u root -p 123456 -t 127.0.0.1 -``` - -Or start your own MySQL server, and then run code coverage - -```shell -$ ./coverage.sh -u ${MYSQL_USERNAME} -p ${MYSQL_PASSWORD} -t ${MYSQL_SERVER_IP} -``` - #### Launch Milvus server ```shell From 090e1ce15e4c4b1a32709a10b2853f9fa730bff7 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 16 Oct 2019 16:34:43 +0800 Subject: [PATCH 34/36] change default db path to /tmp/milvus Former-commit-id: d161cc82ee46ac10460f3a7b8e85a8ac4f0fea46 --- ci/jenkinsfile/milvus_build.groovy | 2 +- ci/jenkinsfile/milvus_build_no_ut.groovy | 2 +- core/build.sh | 20 +++++++++++++------- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/ci/jenkinsfile/milvus_build.groovy b/ci/jenkinsfile/milvus_build.groovy index 7b0da2116c..11f8c51d33 100644 --- a/ci/jenkinsfile/milvus_build.groovy +++ b/ci/jenkinsfile/milvus_build.groovy @@ -15,7 +15,7 @@ container('milvus-build-env') { && export JFROG_USER_NAME='${USERNAME}' \ && export JFROG_PASSWORD='${PASSWORD}' \ && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.2.1/faiss-branch-0.2.1.tar.gz' \ - && ./build.sh -t ${params.BUILD_TYPE} -j -u -c" + && ./build.sh -t ${params.BUILD_TYPE} -d /opt/milvus -j -u -c" sh "./coverage.sh -u root -p 123456 -t 192.168.1.194" } diff --git a/ci/jenkinsfile/milvus_build_no_ut.groovy b/ci/jenkinsfile/milvus_build_no_ut.groovy index 24f2cedef4..1dd3361106 100644 --- a/ci/jenkinsfile/milvus_build_no_ut.groovy +++ b/ci/jenkinsfile/milvus_build_no_ut.groovy @@ -15,7 +15,7 @@ container('milvus-build-env') { && export JFROG_USER_NAME='${USERNAME}' \ && export JFROG_PASSWORD='${PASSWORD}' \ && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.2.1/faiss-branch-0.2.1.tar.gz' \ - && ./build.sh -t ${params.BUILD_TYPE} -j" + && ./build.sh -t ${params.BUILD_TYPE} -j -d /opt/milvus" } } } catch (exc) { diff --git a/core/build.sh b/core/build.sh index c0bd9c5221..795c87c6bf 100755 --- a/core/build.sh +++ b/core/build.sh @@ -1,11 +1,12 @@ #!/bin/bash +BUILD_OUTPUT_DIR="cmake_build" BUILD_TYPE="Debug" BUILD_UNITTEST="OFF" INSTALL_PREFIX=$(pwd)/milvus MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" -DB_PATH="/opt/milvus" +DB_PATH="/tmp/milvus" PROFILING="OFF" USE_JFROG_CACHE="OFF" RUN_CPPLINT="OFF" @@ -40,8 +41,8 @@ do RUN_CPPLINT="ON" ;; r) - if [[ -d cmake_build ]]; then - rm ./cmake_build -r + if [[ -d ${BUILD_OUTPUT_DIR} ]]; then + rm ./${BUILD_OUTPUT_DIR} -r MAKE_CLEAN="ON" fi ;; @@ -62,7 +63,7 @@ do parameter: -p: install prefix(default: $(pwd)/milvus) --d: db path(default: /opt/milvus) +-d: db data path(default: /tmp/milvus) -t: build type(default: Debug) -u: building unit test options(default: OFF) -l: run cpplint, clang-format and clang-tidy(default: OFF) @@ -84,11 +85,11 @@ usage: esac done -if [[ ! -d cmake_build ]]; then - mkdir cmake_build +if [[ ! -d ${BUILD_OUTPUT_DIR} ]]; then + mkdir ${BUILD_OUTPUT_DIR} fi -cd cmake_build +cd ${BUILD_OUTPUT_DIR} CMAKE_CMD="cmake \ -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ @@ -114,6 +115,7 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then make lint if [ $? -ne 0 ]; then echo "ERROR! cpplint check failed" + rm -f CMakeCache.txt exit 1 fi echo "cpplint check passed!" @@ -122,6 +124,7 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then make check-clang-format if [ $? -ne 0 ]; then echo "ERROR! clang-format check failed" + rm -f CMakeCache.txt exit 1 fi echo "clang-format check passed!" @@ -130,9 +133,12 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then # make check-clang-tidy # if [ $? -ne 0 ]; then # echo "ERROR! clang-tidy check failed" +# rm -f CMakeCache.txt # exit 1 # fi # echo "clang-tidy check passed!" + + rm -f CMakeCache.txt else # compile and build make -j 4 || exit 1 From 311897202aca76973ebfa44b989fe63f92953001 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 16 Oct 2019 16:41:49 +0800 Subject: [PATCH 35/36] Update README.md Former-commit-id: 83fd278d294af5578941f7d1e4c8df180773035c --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5ea44c3122..4f57318c90 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Keep up-to-date with newest releases and latest updates by reading Milvus [relea The data is stored and computed on a distributed architecture. This lets you scale data sizes up and down without redesigning the system. ## Architecture -![Milvus_arch](https://github.com/milvus-io/docs/blob/branch-0.5.0/assets/milvus_arch.png) +![Milvus_arch](https://github.com/milvus-io/docs/blob/branch-0.5.0/assets/milvus_arch.) ## Get started From 6ec58ac425444c9c9490fbc4619266387a480f26 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 16 Oct 2019 16:48:43 +0800 Subject: [PATCH 36/36] Update README.md Former-commit-id: c3c92b02c8ad05213dfff8af149cfbbeef2e5a7c --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4f57318c90..08760798e2 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Keep up-to-date with newest releases and latest updates by reading Milvus [relea The data is stored and computed on a distributed architecture. This lets you scale data sizes up and down without redesigning the system. ## Architecture -![Milvus_arch](https://github.com/milvus-io/docs/blob/branch-0.5.0/assets/milvus_arch.) +![Milvus_arch](https://github.com/milvus-io/docs/blob/branch-0.5.0/assets/milvus_arch.jpg) ## Get started