From 5aad000a93e2018352c0816c859a381d93debed7 Mon Sep 17 00:00:00 2001 From: Yinzuo Jiang Date: Fri, 1 Nov 2024 13:08:21 +0800 Subject: [PATCH] enhance: [skip ci] remove tools/core_gen after #35251 (#36306) `tools/core_gen` python scripts are useless after https://github.com/milvus-io/milvus/pull/35251 fixes: #36305 Signed-off-by: Yinzuo Jiang Signed-off-by: Yinzuo Jiang --- .../core/src/query/ExecPlanNodeVisitor.cpp | 3 +- internal/core/src/query/ExecPlanNodeVisitor.h | 2 - tools/core_gen/__init__.py | 0 tools/core_gen/all_generate.py | 112 ------------------ tools/core_gen/assemble.py | 38 ------ tools/core_gen/meta_gen.py | 64 ---------- tools/core_gen/templates/node_def.cpp | 19 --- tools/core_gen/templates/visitor_base.h | 20 ---- tools/core_gen/templates/visitor_derived.cpp | 18 --- tools/core_gen/templates/visitor_derived.h | 24 ---- 10 files changed, 1 insertion(+), 299 deletions(-) delete mode 100644 tools/core_gen/__init__.py delete mode 100755 tools/core_gen/all_generate.py delete mode 100755 tools/core_gen/assemble.py delete mode 100755 tools/core_gen/meta_gen.py delete mode 100644 tools/core_gen/templates/node_def.cpp delete mode 100644 tools/core_gen/templates/visitor_base.h delete mode 100644 tools/core_gen/templates/visitor_derived.cpp delete mode 100644 tools/core_gen/templates/visitor_derived.h diff --git a/internal/core/src/query/ExecPlanNodeVisitor.cpp b/internal/core/src/query/ExecPlanNodeVisitor.cpp index 58d7cbcf12..d00488bf94 100644 --- a/internal/core/src/query/ExecPlanNodeVisitor.cpp +++ b/internal/core/src/query/ExecPlanNodeVisitor.cpp @@ -28,8 +28,7 @@ namespace milvus::query { namespace impl { -// THIS CONTAINS EXTRA BODY FOR VISITOR -// WILL BE USED BY GENERATOR UNDER suvlim/core_gen/ + class ExecPlanNodeVisitor : PlanNodeVisitor { public: ExecPlanNodeVisitor(const segcore::SegmentInterface& segment, diff --git a/internal/core/src/query/ExecPlanNodeVisitor.h b/internal/core/src/query/ExecPlanNodeVisitor.h index 601f12ba3e..1d9f160da9 100644 --- a/internal/core/src/query/ExecPlanNodeVisitor.h +++ b/internal/core/src/query/ExecPlanNodeVisitor.h @@ -10,8 +10,6 @@ // or implied. See the License for the specific language governing permissions and limitations under the License #pragma once -// Generated File -// DO NOT EDIT #include "common/Json.h" #include "query/PlanImpl.h" #include "segcore/SegmentGrowing.h" diff --git a/tools/core_gen/__init__.py b/tools/core_gen/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tools/core_gen/all_generate.py b/tools/core_gen/all_generate.py deleted file mode 100755 index 9d4685a4c5..0000000000 --- a/tools/core_gen/all_generate.py +++ /dev/null @@ -1,112 +0,0 @@ -#!python -# from gen_base_visitor import * -# from gen_node import * -from assemble import * -from meta_gen import * -import re -import os - -def gen_file(rootfile, template, output, **kwargs): - namespace, root_base, struct_name = meta_gen(readfile(rootfile)) - vc = assemble(readfile(template), namespace=namespace, root_base=root_base, struct_name=struct_name, **kwargs) - file = open(output, 'w') - license = open("../../internal/core/build-support/cpp_license.txt").read() - file.write(license + vc) - - -def extract_extra_body(visitor_info, query_path): - pattern = re.compile(r"class(.*){\n((.|\n)*?)\n};", re.MULTILINE) - - for node, visitors in visitor_info.items(): - for visitor in visitors: - vis_name = visitor['visitor_name'] - vis_file = query_path + "visitors/" + vis_name + ".cpp" - body = ' public:' - - inc_pattern_str = r'^(#include(.|\n)*)\n#include "query/generated/{}.h"'.format(vis_name) - inc_pattern = re.compile(inc_pattern_str, re.MULTILINE) - - if os.path.exists(vis_file): - content = readfile(vis_file) - infos = pattern.findall(content) - assert len(infos) <= 1 - if len(infos) == 1: - name, body, _ = infos[0] - - extra_inc_infos = inc_pattern.findall(content) - assert(len(extra_inc_infos) <= 1) - print(extra_inc_infos) - if len(extra_inc_infos) == 1: - extra_inc_body, _ = extra_inc_infos[0] - - visitor["ctor_and_member"] = body - visitor["extra_inc"] = extra_inc_body - -if __name__ == "__main__": - query_path = "../../internal/core/src/query/" - output_path = query_path + "generated/" - - - node_names = ["Expr", "PlanNode"] - visitor_info = { - 'Expr': [ - { - 'visitor_name': "ShowExprVisitor", - "parameter_name": 'expr', - }, - { - 'visitor_name': "ExecExprVisitor", - "parameter_name": 'expr', - }, - { - 'visitor_name': "VerifyExprVisitor", - "parameter_name": 'expr', - }, - { - 'visitor_name': "ExtractInfoExprVisitor", - "parameter_name": 'expr', - }, - ], - 'PlanNode': [ - { - 'visitor_name': "ShowPlanNodeVisitor", - "parameter_name": 'node', - }, - { - 'visitor_name': "ExecPlanNodeVisitor", - "parameter_name": 'node', - }, - { - 'visitor_name': "VerifyPlanNodeVisitor", - "parameter_name": 'node', - }, - { - 'visitor_name': "ExtractInfoPlanNodeVisitor", - "parameter_name": 'node', - }, - ] - } - extract_extra_body(visitor_info, query_path) - - for name in node_names: - rootfile = query_path + name + ".h" - - template = 'templates/visitor_base.h' - output = output_path + name + 'Visitor.h' - gen_file(rootfile, template, output) - - template = 'templates/node_def.cpp' - output = output_path + name + '.cpp' - gen_file(rootfile, template, output) - - for info in visitor_info[name]: - vis_name = info['visitor_name'] - template = 'templates/visitor_derived.h' - output = output_path + vis_name + '.h' - gen_file(rootfile, template, output, **info) - - vis_name = info['visitor_name'] - template = 'templates/visitor_derived.cpp' - output = output_path + vis_name + '.cpp' - gen_file(rootfile, template, output, **info) - print("Done") diff --git a/tools/core_gen/assemble.py b/tools/core_gen/assemble.py deleted file mode 100755 index a9ce6c9cfe..0000000000 --- a/tools/core_gen/assemble.py +++ /dev/null @@ -1,38 +0,0 @@ -#!python -from meta_gen import * -import re - -def assemble(template, **kwargs): - pattern = re.compile("@@@@(.*?)\n((.|\n)*?)\n####", re.MULTILINE) - temp_info = pattern.findall(template) - # print(temp_info) - mapping = dict() - rep_map = dict() - - # drop repetive field from mapping - for k, v in kwargs.items(): - if isinstance(v, list): - rep_map[k] = v - else: - mapping[k] = v - - for k, v, _ in temp_info: - info = k.split("@") - new_v = replace_all(v, **mapping) - assert(1 <= len(info) <= 2) - if len(info) == 2: - k = info[0] - rep = info[1] - new_v = "\n\n".join([new_v.replace("@@" + rep + "@@", rep_v) for rep_v in rep_map[rep]]) - mapping[k] = new_v - return mapping["main"] - - -# import sys -# if __name__ == "__main__": -# assert(len(sys.argv) == 2) -# root_file = sys.argv[1] -# namespace, root_base, struct_name = meta_gen(readfile(root_file)) -# gen_all(readfile("templates/node_full.cpp"), namespace=namespace, root_base=root_base, struct_name=struct_name) - - diff --git a/tools/core_gen/meta_gen.py b/tools/core_gen/meta_gen.py deleted file mode 100755 index 072129157a..0000000000 --- a/tools/core_gen/meta_gen.py +++ /dev/null @@ -1,64 +0,0 @@ -#!python -import re -import sys - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - -def readfile(filename): - file = open(filename) - content = file.read() - return content - -def replace_all(template, **kwargs): - for k, v in kwargs.items(): - template = template.replace("@@" + k + "@@", v) - return template - - -def meta_gen(content): - namespace_pattern = re.compile(r"namespace(.*){") - results = namespace_pattern.findall(content) - assert(len(results) == 1) - namespace = results[0].strip() - - struct_pattern = re.compile(r"struct (.*?){((.|\n)*?)^};", re.MULTILINE) - results = struct_pattern.findall(content) - - body_pattern = re.compile(r"accept\((.*)Visitor ?& ?\) (.*?);") - # print(results) - # print(len(results[0])) - - root_base = None - override_structs = [] - for (title, body, _) in results: - pack = title.replace(' ', '').split(':') - - if len(pack) == 1: - pack.append(None) - - body_res = body_pattern.findall(body) - if len(body_res) != 1: - continue - eprint(struct_name) - eprint(body_res) - eprint(body) - assert False - struct_name, base_name = pack - if not base_name: - root_base = struct_name - visitor_name, state = body_res[0] - assert(visitor_name == root_base) - if state.strip() == 'override': - override_structs.append(struct_name) - # print(body_res) - return namespace, root_base, override_structs - -if __name__ == "__main__": - assert(len(sys.argv) == 2) - file = open(sys.argv[1]) - content = file.read() - namespace, root_base, override_structs = meta_gen(content) - eprint(namespace) - eprint(root_base) - eprint(override_structs) diff --git a/tools/core_gen/templates/node_def.cpp b/tools/core_gen/templates/node_def.cpp deleted file mode 100644 index 81012df36e..0000000000 --- a/tools/core_gen/templates/node_def.cpp +++ /dev/null @@ -1,19 +0,0 @@ -@@@@body@struct_name -void -@@struct_name@@::accept(@@root_base@@Visitor& visitor) { - visitor.visit(*this); -} -#### - -@@@@main -// Generated File -// DO NOT EDIT -#include "query/@@root_base@@.h" -#include "@@root_base@@Visitor.h" - -namespace @@namespace@@ { -@@body@@ - -} // namespace @@namespace@@ - -#### \ No newline at end of file diff --git a/tools/core_gen/templates/visitor_base.h b/tools/core_gen/templates/visitor_base.h deleted file mode 100644 index e210f10a5a..0000000000 --- a/tools/core_gen/templates/visitor_base.h +++ /dev/null @@ -1,20 +0,0 @@ -@@@@body@struct_name - virtual void - visit(@@struct_name@@&) = 0; -#### -@@@@main -#pragma once -// Generated File -// DO NOT EDIT -#include "query/@@root_base@@.h" -namespace @@namespace@@ { -class @@root_base@@Visitor { - public: - virtual ~@@root_base@@Visitor() = default; - - public: -@@body@@ -}; -} // namespace @@namespace@@ - -#### \ No newline at end of file diff --git a/tools/core_gen/templates/visitor_derived.cpp b/tools/core_gen/templates/visitor_derived.cpp deleted file mode 100644 index a6a5a7000e..0000000000 --- a/tools/core_gen/templates/visitor_derived.cpp +++ /dev/null @@ -1,18 +0,0 @@ -@@@@func_list@struct_name -void -@@visitor_name@@::visit(@@struct_name@@& @@parameter_name@@) { - // TODO -} -#### - - -@@@@main -#error TODO: copy this file out, and modify the content. -#include "query/generated/@@visitor_name@@.h" - -namespace @@namespace@@ { -@@func_list@@ - -} // namespace @@namespace@@ - -#### diff --git a/tools/core_gen/templates/visitor_derived.h b/tools/core_gen/templates/visitor_derived.h deleted file mode 100644 index 5fe2be775d..0000000000 --- a/tools/core_gen/templates/visitor_derived.h +++ /dev/null @@ -1,24 +0,0 @@ -@@@@base_visitor -@@root_base@@Visitor -#### -@@@@body@struct_name - void - visit(@@struct_name@@& @@parameter_name@@) override; -#### -@@@@main -#pragma once -// Generated File -// DO NOT EDIT -@@extra_inc@@ -#include "@@base_visitor@@.h" - -namespace @@namespace@@ { -class @@visitor_name@@ : public @@base_visitor@@ { - public: -@@body@@ - -@@ctor_and_member@@ -}; -} // namespace @@namespace@@ - -####