milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

#4066 Rewrite SqliteMetaImpl to discard sqlite_orm (#4114) 

* #4066 Rewrite SqliteMetaImpl to discard sqlite_orm

Signed-off-by: yhmo <yihua.mo@zilliz.com>

* fix unittest

Signed-off-by: yhmo <yihua.mo@zilliz.com>
pull/4129/head
groot 2020-10-29 15:29:02 +08:00 committed by GitHub
parent 84083294c3
commit 762f5f296f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 1485 additions and 1002 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
core/src/db/meta/MetaSchema.cpp Normal file
View File

@ -0,0 +1,67 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "db/meta/MetaSchema.h"
#include "utils/StringHelpFunctions.h"
namespace milvus::engine::meta {
bool
MetaField::IsEqual(const MetaField& field) const {
// mysql field type has additional information. for instance, a filed type is defined as 'BIGINT'
// we get the type from sql is 'bigint(20)', so we need to ignore the '(20)'
size_t name_len_min = field.name_.length() > name_.length() ? name_.length() : field.name_.length();
size_t type_len_min = field.type_.length() > type_.length() ? type_.length() : field.type_.length();
// only check field type, don't check field width, for example: VARCHAR(255) and VARCHAR(100) is equal
std::vector<std::string> type_split;
milvus::server::StringHelpFunctions::SplitStringByDelimeter(type_, "(", type_split);
if (!type_split.empty()) {
type_len_min = type_split[0].length() > type_len_min ? type_len_min : type_split[0].length();
}
// field name must be equal, ignore type width
return strncasecmp(field.name_.c_str(), name_.c_str(), name_len_min) == 0 &&
strncasecmp(field.type_.c_str(), type_.c_str(), type_len_min) == 0;
}
std::string
MetaSchema::ToString() const {
std::string result;
for (auto& field : fields_) {
if (!result.empty()) {
result += ",";
}
result += field.ToString();
}
return result;
}
// if the outer fields contains all this MetaSchema fields, return true
// otherwise return false
bool
MetaSchema::IsEqual(const MetaFields& fields) const {
std::vector<std::string> found_field;
for (const auto& this_field : fields_) {
for (const auto& outer_field : fields) {
if (this_field.IsEqual(outer_field)) {
found_field.push_back(this_field.name());
break;
}
}
}
return found_field.size() == fields_.size();
}
} // namespace milvus::engine::meta

69
core/src/db/meta/MetaSchema.h Normal file
View File

@ -0,0 +1,69 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <unistd.h>
#include <cstring>
#include <string>
#include <vector>
namespace milvus::engine::meta {
class MetaField {
public:
MetaField(const std::string& name, const std::string& type, const std::string& setting)
: name_(name), type_(type), setting_(setting) {
}
const std::string&
name() const {
return name_;
}
std::string
ToString() const {
return name_ + " " + type_ + " " + setting_;
}
bool
IsEqual(const MetaField& field) const;
private:
std::string name_;
std::string type_;
std::string setting_;
};
using MetaFields = std::vector<MetaField>;
class MetaSchema {
public:
MetaSchema(const std::string& name, const MetaFields& fields) : name_(name), fields_(fields) {
}
std::string
name() const {
return name_;
}
std::string
ToString() const;
bool
IsEqual(const MetaFields& fields) const;
private:
std::string name_;
MetaFields fields_;
};
} // namespace milvus::engine::meta

88
core/src/db/meta/MySQLMetaImpl.cpp
View File

@ -33,6 +33,7 @@
#include "MetaConsts.h"
#include "db/IDGenerator.h"
#include "db/Utils.h"
#include "db/meta/MetaSchema.h"
#include "metrics/Metrics.h"
#include "utils/CommonUtil.h"
#include "utils/Exception.h"
@ -78,93 +79,6 @@ HandleException(const std::string& desc, const char* what = nullptr) {
return Status(DB_META_TRANSACTION_FAILED, msg);
}
class MetaField {
public:
MetaField(const std::string& name, const std::string& type, const std::string& setting)
: name_(name), type_(type), setting_(setting) {
}
std::string
name() const {
return name_;
}
std::string
ToString() const {
return name_ + " " + type_ + " " + setting_;
}
// mysql field type has additional information. for instance, a filed type is defined as 'BIGINT'
// we get the type from sql is 'bigint(20)', so we need to ignore the '(20)'
bool
IsEqual(const MetaField& field) const {
size_t name_len_min = field.name_.length() > name_.length() ? name_.length() : field.name_.length();
size_t type_len_min = field.type_.length() > type_.length() ? type_.length() : field.type_.length();
// only check field type, don't check field width, for example: VARCHAR(255) and VARCHAR(100) is equal
std::vector<std::string> type_split;
milvus::server::StringHelpFunctions::SplitStringByDelimeter(type_, "(", type_split);
if (!type_split.empty()) {
type_len_min = type_split[0].length() > type_len_min ? type_len_min : type_split[0].length();
}
// field name must be equal, ignore type width
return strncasecmp(field.name_.c_str(), name_.c_str(), name_len_min) == 0 &&
strncasecmp(field.type_.c_str(), type_.c_str(), type_len_min) == 0;
}
private:
std::string name_;
std::string type_;
std::string setting_;
};
using MetaFields = std::vector<MetaField>;
class MetaSchema {
public:
MetaSchema(const std::string& name, const MetaFields& fields) : name_(name), fields_(fields) {
}
std::string
name() const {
return name_;
}
std::string
ToString() const {
std::string result;
for (auto& field : fields_) {
if (!result.empty()) {
result += ",";
}
result += field.ToString();
}
return result;
}
// if the outer fields contains all this MetaSchema fields, return true
// otherwise return false
bool
IsEqual(const MetaFields& fields) const {
std::vector<std::string> found_field;
for (const auto& this_field : fields_) {
for (const auto& outer_field : fields) {
if (this_field.IsEqual(outer_field)) {
found_field.push_back(this_field.name());
break;
}
}
}
return found_field.size() == fields_.size();
}
private:
std::string name_;
MetaFields fields_;
};
// Environment schema
static const MetaSchema ENVIRONMENT_SCHEMA(META_ENVIRONMENT, {
MetaField("global_lsn", "BIGINT", "NOT NULL"),

2240
core/src/db/meta/SqliteMetaImpl.cpp
View File

File diff suppressed because it is too large Load Diff

21
core/src/db/meta/SqliteMetaImpl.h
View File

@ -14,20 +14,20 @@
#include <mutex>
#include <set>
#include <string>
#include <unordered_map>
#include <vector>
#include "Meta.h"
#include "db/Options.h"
#include <sqlite3.h>
namespace milvus {
namespace engine {
namespace meta {
auto
StoragePrototype(const std::string& path);
auto
CollectionPrototype(const std::string& path);
using AttrsMap = std::unordered_map<std::string, std::string>;
using AttrsMapList = std::vector<AttrsMap>;
class SqliteMetaImpl : public Meta {
public:
@ -179,10 +179,19 @@ class SqliteMetaImpl : public Meta {
Status
Initialize();
Status
SqlQuery(const std::string& sql, AttrsMapList* res = nullptr);
Status
SqlTransaction(const std::vector<std::string>& sql_statements);
private:
const DBMetaOptions options_;
std::mutex meta_mutex_;
std::mutex sqlite_mutex_; // make sqlite query/execute action to be atomic
std::mutex operation_mutex_; // make operation such UpdateTableFiles to be atomic
std::mutex genid_mutex_;
sqlite3* db_ = nullptr;
}; // DBMetaImpl
} // namespace meta

2
core/unittest/db/test_db.cpp
View File

@ -200,7 +200,7 @@ TEST_F(DBTest, DB_TEST) {
std::vector<std::string> tags;
stat = db_->Query(dummy_context_, COLLECTION_NAME, tags, k, json_params, qxb, result_ids, result_distances);
ss << "Search " << j << " With Size " << count / milvus::engine::MB << " MB";
ss << "Search " << j << " With Size " << count / milvus::engine::MB << " MB" << stat.message();
STOP_TIMER(ss.str());
ASSERT_TRUE(stat.ok());