mirror of https://github.com/milvus-io/milvus.git
Fix bug for cmake_build dir was changed
Signed-off-by: cai.zhang <cai.zhang@zilliz.com>pull/4973/head^2
parent
205abfbbcd
commit
3b2c86c42d
|
@ -6,7 +6,6 @@ import (
|
|||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
|
@ -25,11 +24,7 @@ func main() {
|
|||
|
||||
// Creates server.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
etcdAddr := conf.Config.Etcd.Address
|
||||
etcdAddr += ":"
|
||||
etcdAddr += strconv.FormatInt(int64(conf.Config.Etcd.Port), 10)
|
||||
|
||||
svr, err := master.CreateServer(ctx, conf.Config.Etcd.Rootpath, conf.Config.Etcd.Rootpath, conf.Config.Etcd.Rootpath, []string{etcdAddr})
|
||||
svr, err := master.CreateServer(ctx)
|
||||
if err != nil {
|
||||
log.Print("create server failed", zap.Error(err))
|
||||
}
|
||||
|
@ -47,9 +42,7 @@ func main() {
|
|||
cancel()
|
||||
}()
|
||||
|
||||
grpcPort := int64(conf.Config.Master.Port)
|
||||
|
||||
if err := svr.Run(grpcPort); err != nil {
|
||||
if err := svr.Run(); err != nil {
|
||||
log.Fatal("run server failed", zap.Error(err))
|
||||
}
|
||||
|
||||
|
|
|
@ -21,7 +21,7 @@ master:
|
|||
|
||||
etcd:
|
||||
address: localhost
|
||||
port: 2379
|
||||
port: 12379
|
||||
rootpath: by-dev
|
||||
segthreshold: 10000
|
||||
|
||||
|
|
|
@ -3,11 +3,11 @@ version: '3.5'
|
|||
services:
|
||||
etcd:
|
||||
image: quay.io/coreos/etcd:latest
|
||||
command: etcd -listen-peer-urls=http://127.0.0.1:2380 -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001 -initial-advertise-peer-urls=http://127.0.0.1:2380 --initial-cluster default=http://127.0.0.1:2380
|
||||
command: etcd -listen-peer-urls=http://127.0.0.1:12380 -advertise-client-urls=http://127.0.0.1:12379 -listen-client-urls http://0.0.0.0:12379,http://0.0.0.0:14001 -initial-advertise-peer-urls=http://127.0.0.1:12380 --initial-cluster default=http://127.0.0.1:12380
|
||||
ports:
|
||||
- "2379:2379"
|
||||
- "2380:2380"
|
||||
- "4001:4001"
|
||||
- "12379:12379"
|
||||
- "12380:12380"
|
||||
- "14001:14001"
|
||||
|
||||
pulsar:
|
||||
image: apachepulsar/pulsar:latest
|
||||
|
@ -16,44 +16,44 @@ services:
|
|||
- "6650:6650"
|
||||
- "18080:8080"
|
||||
|
||||
# pd0:
|
||||
# image: pingcap/pd:latest
|
||||
# network_mode: "host"
|
||||
# ports:
|
||||
# - "2379:2379"
|
||||
# - "2380:2380"
|
||||
# volumes:
|
||||
# - /tmp/config/pd.toml:/pd.toml:ro
|
||||
# - /tmp/data:/data
|
||||
# - /tmp/logs:/logs
|
||||
# - /etc/localtime:/etc/localtime:ro
|
||||
# command:
|
||||
# - --name=pd0
|
||||
# - --client-urls=http://0.0.0.0:2379
|
||||
# - --peer-urls=http://0.0.0.0:2380
|
||||
# - --advertise-client-urls=http://127.0.0.1:2379
|
||||
# - --advertise-peer-urls=http://127.0.0.1:2380
|
||||
# - --initial-cluster=pd0=http://127.0.0.1:2380
|
||||
# - --data-dir=/data/pd0
|
||||
# - --log-file=/logs/pd0.log
|
||||
# restart: on-failure
|
||||
#
|
||||
# tikv0:
|
||||
# network_mode: "host"
|
||||
# image: pingcap/tikv:latest
|
||||
# ports:
|
||||
# - "20160:20160"
|
||||
# volumes:
|
||||
# - /tmp/config/tikv.toml:/tikv.toml:ro
|
||||
# - /tmp/data:/data
|
||||
# - /tmp/logs:/logs
|
||||
# - /etc/localtime:/etc/localtime:ro
|
||||
# command:
|
||||
# - --addr=0.0.0.0:20160
|
||||
# - --advertise-addr=127.0.0.1:20160
|
||||
# - --data-dir=/data/tikv0
|
||||
# - --pd=127.0.0.1:2379
|
||||
# - --log-file=/logs/tikv0.log
|
||||
# depends_on:
|
||||
# - "pd0"
|
||||
# restart: on-failure
|
||||
pd0:
|
||||
image: pingcap/pd:latest
|
||||
network_mode: "host"
|
||||
ports:
|
||||
- "2379:2379"
|
||||
- "2380:2380"
|
||||
volumes:
|
||||
- /tmp/config/pd.toml:/pd.toml:ro
|
||||
- /tmp/data:/data
|
||||
- /tmp/logs:/logs
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
command:
|
||||
- --name=pd0
|
||||
- --client-urls=http://0.0.0.0:2379
|
||||
- --peer-urls=http://0.0.0.0:2380
|
||||
- --advertise-client-urls=http://127.0.0.1:2379
|
||||
- --advertise-peer-urls=http://127.0.0.1:2380
|
||||
- --initial-cluster=pd0=http://127.0.0.1:2380
|
||||
- --data-dir=/data/pd0
|
||||
- --log-file=/logs/pd0.log
|
||||
restart: on-failure
|
||||
|
||||
tikv0:
|
||||
network_mode: "host"
|
||||
image: pingcap/tikv:latest
|
||||
ports:
|
||||
- "20160:20160"
|
||||
volumes:
|
||||
- /tmp/config/tikv.toml:/tikv.toml:ro
|
||||
- /tmp/data:/data
|
||||
- /tmp/logs:/logs
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
command:
|
||||
- --addr=0.0.0.0:20160
|
||||
- --advertise-addr=127.0.0.1:20160
|
||||
- --data-dir=/data/tikv0
|
||||
- --pd=127.0.0.1:2379
|
||||
- --log-file=/logs/tikv0.log
|
||||
depends_on:
|
||||
- "pd0"
|
||||
restart: on-failure
|
||||
|
|
|
@ -1169,15 +1169,14 @@ type softTimeTickBarrier struct {
|
|||
minTtInterval Timestamp
|
||||
lastTt Timestamp
|
||||
outTt chan Timestamp
|
||||
ttStream MsgStream
|
||||
ttStream *MsgStream
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func (ttBarrier *softTimeTickBarrier) GetTimeTick() (Timestamp,error)
|
||||
func (ttBarrier *softTimeTickBarrier) Start() error
|
||||
func (ttBarrier *softTimeTickBarrier) Close()
|
||||
|
||||
func NewSoftTimeTickBarrier(ctx context.Context, ttStream *MsgStream, peerIds []UniqueID, minTtInterval Timestamp) *softTimeTickBarrier
|
||||
func newSoftTimeTickBarrier(ctx context.Context, ttStream *MsgStream, peerIds []UniqueId, minTtInterval Timestamp) *softTimeTickBarrier
|
||||
```
|
||||
|
||||
|
||||
|
@ -1190,15 +1189,14 @@ func NewSoftTimeTickBarrier(ctx context.Context, ttStream *MsgStream, peerIds []
|
|||
type hardTimeTickBarrier struct {
|
||||
peer2Tt map[UniqueId]List
|
||||
outTt chan Timestamp
|
||||
ttStream MsgStream
|
||||
ttStream *MsgStream
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func (ttBarrier *hardTimeTickBarrier) GetTimeTick() (Timestamp,error)
|
||||
func (ttBarrier *hardTimeTickBarrier) Start() error
|
||||
func (ttBarrier *hardTimeTickBarrier) Close()
|
||||
|
||||
func NewHardTimeTickBarrier(ctx context.Context, ttStream *MsgStream, peerIds []UniqueID) *hardTimeTickBarrier
|
||||
func newHardTimeTickBarrier(ctx context.Context, ttStream *MsgStream, peerIds []UniqueId) *softTimeTickBarrier
|
||||
```
|
||||
|
||||
|
||||
|
@ -1212,7 +1210,6 @@ func NewHardTimeTickBarrier(ctx context.Context, ttStream *MsgStream, peerIds []
|
|||
type TimeTickBarrier interface {
|
||||
GetTimeTick() (Timestamp,error)
|
||||
Start() error
|
||||
Close()
|
||||
}
|
||||
|
||||
type timeSyncMsgProducer struct {
|
||||
|
|
1
go.mod
1
go.mod
|
@ -9,6 +9,7 @@ require (
|
|||
github.com/coreos/etcd v3.3.25+incompatible // indirect
|
||||
github.com/frankban/quicktest v1.10.2 // indirect
|
||||
github.com/fsnotify/fsnotify v1.4.9 // indirect
|
||||
github.com/gogo/protobuf v1.3.1
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
|
||||
github.com/golang/protobuf v1.3.2
|
||||
github.com/google/btree v1.0.0
|
||||
|
|
11
go.sum
11
go.sum
|
@ -15,7 +15,6 @@ github.com/aws/aws-sdk-go v1.30.8 h1:4BHbh8K3qKmcnAgToZ2LShldRF9inoqIBccpCLNCy3I
|
|||
github.com/aws/aws-sdk-go v1.30.8/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
|
||||
github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
|
@ -69,8 +68,9 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB
|
|||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
|
||||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
||||
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
|
||||
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
|
@ -133,6 +133,7 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
|
|||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.10.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||
github.com/klauspost/compress v1.10.11 h1:K9z59aO18Aywg2b/WSgBaUX99mHy2BES18Cr5lBKZHk=
|
||||
|
@ -195,7 +196,6 @@ github.com/ozonru/etcd v3.3.20-grpc1.27-origmodule+incompatible/go.mod h1:iIubIL
|
|||
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI=
|
||||
github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg=
|
||||
github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
|
||||
github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 h1:R8gStypOBmpnHEx1qi//SaqxJVI4inOqljg/Aj5/390=
|
||||
github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc=
|
||||
|
@ -223,25 +223,21 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
|
|||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
|
||||
github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
|
||||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA=
|
||||
github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
|
||||
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
|
||||
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
|
||||
github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
|
||||
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
|
||||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
||||
github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
|
||||
|
@ -391,6 +387,7 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqG
|
|||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
|
|
|
@ -22,4 +22,3 @@ go run cmd/master.go
|
|||
### example
|
||||
if master create a collection with uuid ```46e468ee-b34a-419d-85ed-80c56bfa4e90```
|
||||
the corresponding key in etcd is $(ETCD_ROOT_PATH)/collection/46e468ee-b34a-419d-85ed-80c56bfa4e90
|
||||
|
||||
|
|
|
@ -5,7 +5,7 @@ import (
|
|||
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/gogo/protobuf/proto"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
|
||||
|
|
|
@ -1,11 +1,11 @@
|
|||
package master
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"strconv"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
|
@ -60,41 +60,53 @@ func (t *createCollectionTask) Ts() (Timestamp, error) {
|
|||
if t.req == nil {
|
||||
return 0, errors.New("null request")
|
||||
}
|
||||
return t.req.Timestamp, nil
|
||||
return Timestamp(t.req.Timestamp), nil
|
||||
}
|
||||
|
||||
func (t *createCollectionTask) Execute() error {
|
||||
if t.req == nil {
|
||||
_ = t.Notify()
|
||||
return errors.New("null request")
|
||||
}
|
||||
|
||||
var schema schemapb.CollectionSchema
|
||||
err := proto.UnmarshalMerge(t.req.Schema.Value, &schema)
|
||||
err := json.Unmarshal(t.req.Schema.Value, &schema)
|
||||
if err != nil {
|
||||
return err
|
||||
_ = t.Notify()
|
||||
return errors.New("unmarshal CollectionSchema failed")
|
||||
}
|
||||
|
||||
collectionId, err := allocGlobalId()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ts, err := t.Ts()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// TODO: allocate collection id
|
||||
var collectionId UniqueID = 0
|
||||
// TODO: allocate timestamp
|
||||
var collectionCreateTime Timestamp = 0
|
||||
|
||||
collection := etcdpb.CollectionMeta{
|
||||
Id: collectionId,
|
||||
Schema: &schema,
|
||||
CreateTime: ts,
|
||||
CreateTime: collectionCreateTime,
|
||||
// TODO: initial segment?
|
||||
SegmentIds: make([]UniqueID, 0),
|
||||
// TODO: initial partition?
|
||||
PartitionTags: make([]string, 0),
|
||||
}
|
||||
|
||||
return t.mt.AddCollection(&collection)
|
||||
collectionJson, err := json.Marshal(&collection)
|
||||
if err != nil {
|
||||
_ = t.Notify()
|
||||
return errors.New("marshal collection failed")
|
||||
}
|
||||
|
||||
err = (*t.kvBase).Save(collectionMetaPrefix+strconv.FormatInt(collectionId, 10), string(collectionJson))
|
||||
if err != nil {
|
||||
_ = t.Notify()
|
||||
return errors.New("save collection failed")
|
||||
}
|
||||
|
||||
t.mt.collId2Meta[collectionId] = collection
|
||||
|
||||
_ = t.Notify()
|
||||
return nil
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
|
@ -115,12 +127,14 @@ func (t *dropCollectionTask) Ts() (Timestamp, error) {
|
|||
|
||||
func (t *dropCollectionTask) Execute() error {
|
||||
if t.req == nil {
|
||||
_ = t.Notify()
|
||||
return errors.New("null request")
|
||||
}
|
||||
|
||||
collectionName := t.req.CollectionName.CollectionName
|
||||
collectionMeta, err := t.mt.GetCollectionByName(collectionName)
|
||||
if err != nil {
|
||||
_ = t.Notify()
|
||||
return err
|
||||
}
|
||||
|
||||
|
@ -128,11 +142,13 @@ func (t *dropCollectionTask) Execute() error {
|
|||
|
||||
err = (*t.kvBase).Remove(collectionMetaPrefix + strconv.FormatInt(collectionId, 10))
|
||||
if err != nil {
|
||||
return err
|
||||
_ = t.Notify()
|
||||
return errors.New("save collection failed")
|
||||
}
|
||||
|
||||
delete(t.mt.collId2Meta, collectionId)
|
||||
|
||||
_ = t.Notify()
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -154,6 +170,7 @@ func (t *hasCollectionTask) Ts() (Timestamp, error) {
|
|||
|
||||
func (t *hasCollectionTask) Execute() error {
|
||||
if t.req == nil {
|
||||
_ = t.Notify()
|
||||
return errors.New("null request")
|
||||
}
|
||||
|
||||
|
@ -163,6 +180,7 @@ func (t *hasCollectionTask) Execute() error {
|
|||
t.hasCollection = true
|
||||
}
|
||||
|
||||
_ = t.Notify()
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -184,12 +202,14 @@ func (t *describeCollectionTask) Ts() (Timestamp, error) {
|
|||
|
||||
func (t *describeCollectionTask) Execute() error {
|
||||
if t.req == nil {
|
||||
_ = t.Notify()
|
||||
return errors.New("null request")
|
||||
}
|
||||
|
||||
collectionName := t.req.CollectionName
|
||||
collection, err := t.mt.GetCollectionByName(collectionName.CollectionName)
|
||||
if err != nil {
|
||||
_ = t.Notify()
|
||||
return err
|
||||
}
|
||||
|
||||
|
@ -202,6 +222,7 @@ func (t *describeCollectionTask) Execute() error {
|
|||
|
||||
t.description = &description
|
||||
|
||||
_ = t.Notify()
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -223,6 +244,7 @@ func (t *showCollectionsTask) Ts() (Timestamp, error) {
|
|||
|
||||
func (t *showCollectionsTask) Execute() error {
|
||||
if t.req == nil {
|
||||
_ = t.Notify()
|
||||
return errors.New("null request")
|
||||
}
|
||||
|
||||
|
@ -240,5 +262,6 @@ func (t *showCollectionsTask) Execute() error {
|
|||
|
||||
t.stringListResponse = &stringListResponse
|
||||
|
||||
_ = t.Notify()
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -2,7 +2,6 @@ package master
|
|||
|
||||
import (
|
||||
"context"
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/tso"
|
||||
"time"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/errors"
|
||||
|
@ -19,13 +18,14 @@ func (s *Master) CreateCollection(ctx context.Context, in *internalpb.CreateColl
|
|||
req: in,
|
||||
baseTask: baseTask{
|
||||
kvBase: s.kvBase,
|
||||
mt: s.mt,
|
||||
cv: make(chan error),
|
||||
mt: &s.mt,
|
||||
cv: make(chan int),
|
||||
},
|
||||
}
|
||||
|
||||
var err = s.scheduler.Enqueue(t)
|
||||
var err = s.scheduler.Enqueue(&t)
|
||||
if err != nil {
|
||||
err := errors.New("Enqueue failed")
|
||||
return &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "Enqueue failed",
|
||||
|
@ -34,9 +34,10 @@ func (s *Master) CreateCollection(ctx context.Context, in *internalpb.CreateColl
|
|||
|
||||
err = t.WaitToFinish(ctx)
|
||||
if err != nil {
|
||||
err := errors.New("WaitToFinish failed")
|
||||
return &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "create collection failed",
|
||||
Reason: "WaitToFinish failed",
|
||||
}, err
|
||||
}
|
||||
|
||||
|
@ -50,12 +51,12 @@ func (s *Master) DropCollection(ctx context.Context, in *internalpb.DropCollecti
|
|||
req: in,
|
||||
baseTask: baseTask{
|
||||
kvBase: s.kvBase,
|
||||
mt: s.mt,
|
||||
cv: make(chan error),
|
||||
mt: &s.mt,
|
||||
cv: make(chan int),
|
||||
},
|
||||
}
|
||||
|
||||
var err = s.scheduler.Enqueue(t)
|
||||
var err = s.scheduler.Enqueue(&t)
|
||||
if err != nil {
|
||||
err := errors.New("Enqueue failed")
|
||||
return &commonpb.Status{
|
||||
|
@ -83,13 +84,13 @@ func (s *Master) HasCollection(ctx context.Context, in *internalpb.HasCollection
|
|||
req: in,
|
||||
baseTask: baseTask{
|
||||
kvBase: s.kvBase,
|
||||
mt: s.mt,
|
||||
cv: make(chan error),
|
||||
mt: &s.mt,
|
||||
cv: make(chan int),
|
||||
},
|
||||
hasCollection: false,
|
||||
}
|
||||
|
||||
var err = s.scheduler.Enqueue(t)
|
||||
var err = s.scheduler.Enqueue(&t)
|
||||
if err != nil {
|
||||
err := errors.New("Enqueue failed")
|
||||
return &servicepb.BoolResponse{
|
||||
|
@ -126,13 +127,13 @@ func (s *Master) DescribeCollection(ctx context.Context, in *internalpb.Describe
|
|||
req: in,
|
||||
baseTask: baseTask{
|
||||
kvBase: s.kvBase,
|
||||
mt: s.mt,
|
||||
cv: make(chan error),
|
||||
mt: &s.mt,
|
||||
cv: make(chan int),
|
||||
},
|
||||
description: nil,
|
||||
}
|
||||
|
||||
var err = s.scheduler.Enqueue(t)
|
||||
var err = s.scheduler.Enqueue(&t)
|
||||
if err != nil {
|
||||
err := errors.New("Enqueue failed")
|
||||
return t.(*describeCollectionTask).description, err
|
||||
|
@ -152,13 +153,13 @@ func (s *Master) ShowCollections(ctx context.Context, in *internalpb.ShowCollect
|
|||
req: in,
|
||||
baseTask: baseTask{
|
||||
kvBase: s.kvBase,
|
||||
mt: s.mt,
|
||||
cv: make(chan error),
|
||||
mt: &s.mt,
|
||||
cv: make(chan int),
|
||||
},
|
||||
stringListResponse: nil,
|
||||
}
|
||||
|
||||
var err = s.scheduler.Enqueue(t)
|
||||
var err = s.scheduler.Enqueue(&t)
|
||||
if err != nil {
|
||||
err := errors.New("Enqueue failed")
|
||||
return t.(*showCollectionsTask).stringListResponse, err
|
||||
|
@ -179,12 +180,12 @@ func (s *Master) CreatePartition(ctx context.Context, in *internalpb.CreateParti
|
|||
req: in,
|
||||
baseTask: baseTask{
|
||||
kvBase: s.kvBase,
|
||||
mt: s.mt,
|
||||
cv: make(chan error),
|
||||
mt: &s.mt,
|
||||
cv: make(chan int),
|
||||
},
|
||||
}
|
||||
|
||||
var err = s.scheduler.Enqueue(t)
|
||||
var err = s.scheduler.Enqueue(&t)
|
||||
if err != nil {
|
||||
err := errors.New("Enqueue failed")
|
||||
return &commonpb.Status{
|
||||
|
@ -212,12 +213,12 @@ func (s *Master) DropPartition(ctx context.Context, in *internalpb.DropPartition
|
|||
req: in,
|
||||
baseTask: baseTask{
|
||||
kvBase: s.kvBase,
|
||||
mt: s.mt,
|
||||
cv: make(chan error),
|
||||
mt: &s.mt,
|
||||
cv: make(chan int),
|
||||
},
|
||||
}
|
||||
|
||||
var err = s.scheduler.Enqueue(t)
|
||||
var err = s.scheduler.Enqueue(&t)
|
||||
if err != nil {
|
||||
err := errors.New("Enqueue failed")
|
||||
return &commonpb.Status{
|
||||
|
@ -245,13 +246,13 @@ func (s *Master) HasPartition(ctx context.Context, in *internalpb.HasPartitionRe
|
|||
req: in,
|
||||
baseTask: baseTask{
|
||||
kvBase: s.kvBase,
|
||||
mt: s.mt,
|
||||
cv: make(chan error),
|
||||
mt: &s.mt,
|
||||
cv: make(chan int),
|
||||
},
|
||||
hasPartition: false,
|
||||
}
|
||||
|
||||
var err = s.scheduler.Enqueue(t)
|
||||
var err = s.scheduler.Enqueue(&t)
|
||||
if err != nil {
|
||||
err := errors.New("Enqueue failed")
|
||||
return &servicepb.BoolResponse{
|
||||
|
@ -288,13 +289,13 @@ func (s *Master) DescribePartition(ctx context.Context, in *internalpb.DescribeP
|
|||
req: in,
|
||||
baseTask: baseTask{
|
||||
kvBase: s.kvBase,
|
||||
mt: s.mt,
|
||||
cv: make(chan error),
|
||||
mt: &s.mt,
|
||||
cv: make(chan int),
|
||||
},
|
||||
description: nil,
|
||||
}
|
||||
|
||||
var err = s.scheduler.Enqueue(t)
|
||||
var err = s.scheduler.Enqueue(&t)
|
||||
if err != nil {
|
||||
err := errors.New("Enqueue failed")
|
||||
return t.(*describePartitionTask).description, err
|
||||
|
@ -314,13 +315,13 @@ func (s *Master) ShowPartitions(ctx context.Context, in *internalpb.ShowPartitio
|
|||
req: in,
|
||||
baseTask: baseTask{
|
||||
kvBase: s.kvBase,
|
||||
mt: s.mt,
|
||||
cv: make(chan error),
|
||||
mt: &s.mt,
|
||||
cv: make(chan int),
|
||||
},
|
||||
stringListResponse: nil,
|
||||
}
|
||||
|
||||
var err = s.scheduler.Enqueue(t)
|
||||
var err = s.scheduler.Enqueue(&t)
|
||||
if err != nil {
|
||||
err := errors.New("Enqueue failed")
|
||||
return t.(*showPartitionTask).stringListResponse, err
|
||||
|
@ -339,7 +340,7 @@ func (s *Master) ShowPartitions(ctx context.Context, in *internalpb.ShowPartitio
|
|||
|
||||
func (s *Master) AllocTimestamp(ctx context.Context, request *internalpb.TsoRequest) (*internalpb.TsoResponse, error) {
|
||||
count := request.GetCount()
|
||||
ts, err := tso.Alloc(count)
|
||||
ts, err := s.tsoAllocator.GenerateTSO(count)
|
||||
|
||||
if err != nil {
|
||||
return &internalpb.TsoResponse{
|
||||
|
|
|
@ -1,140 +0,0 @@
|
|||
package master
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/masterpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
|
||||
"go.etcd.io/etcd/clientv3"
|
||||
"google.golang.org/grpc"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMaster_CreateCollection(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.TODO())
|
||||
defer cancel()
|
||||
|
||||
etcdCli, err := clientv3.New(clientv3.Config{Endpoints: []string{"127.0.0.1:2379"}})
|
||||
assert.Nil(t, err)
|
||||
_, err = etcdCli.Delete(ctx, "/test/root", clientv3.WithPrefix())
|
||||
assert.Nil(t, err)
|
||||
|
||||
svr, err := CreateServer(ctx, "/test/root/kv", "/test/root/meta", "/test/root/meta/tso", []string{"127.0.0.1:2379"})
|
||||
assert.Nil(t, err)
|
||||
err = svr.Run(10001)
|
||||
assert.Nil(t, err)
|
||||
|
||||
conn, err := grpc.DialContext(ctx, "127.0.0.1:10001", grpc.WithInsecure(), grpc.WithBlock())
|
||||
assert.Nil(t, err)
|
||||
defer conn.Close()
|
||||
|
||||
cli := masterpb.NewMasterClient(conn)
|
||||
sch := schemapb.CollectionSchema{
|
||||
Name: "col1",
|
||||
Description: "test collection",
|
||||
AutoId: false,
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{
|
||||
Name: "col1_f1",
|
||||
Description: "test collection filed 1",
|
||||
DataType: schemapb.DataType_VECTOR_FLOAT,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "col1_f1_tk1",
|
||||
Value: "col1_f1_tv1",
|
||||
},
|
||||
{
|
||||
Key: "col1_f1_tk2",
|
||||
Value: "col1_f1_tv2",
|
||||
},
|
||||
},
|
||||
IndexParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "col1_f1_ik1",
|
||||
Value: "col1_f1_iv1",
|
||||
},
|
||||
{
|
||||
Key: "col1_f1_ik2",
|
||||
Value: "col1_f1_iv2",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "col1_f2",
|
||||
Description: "test collection filed 2",
|
||||
DataType: schemapb.DataType_VECTOR_BINARY,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "col1_f2_tk1",
|
||||
Value: "col1_f2_tv1",
|
||||
},
|
||||
{
|
||||
Key: "col1_f2_tk2",
|
||||
Value: "col1_f2_tv2",
|
||||
},
|
||||
},
|
||||
IndexParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "col1_f2_ik1",
|
||||
Value: "col1_f2_iv1",
|
||||
},
|
||||
{
|
||||
Key: "col1_f2_ik2",
|
||||
Value: "col1_f2_iv2",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
schema_bytes, err := proto.Marshal(&sch)
|
||||
assert.Nil(t, err)
|
||||
|
||||
req := internalpb.CreateCollectionRequest{
|
||||
MsgType: internalpb.MsgType_kCreateCollection,
|
||||
ReqId: 1,
|
||||
Timestamp: 11,
|
||||
ProxyId: 1,
|
||||
Schema: &commonpb.Blob{Value: schema_bytes},
|
||||
}
|
||||
st, err := cli.CreateCollection(ctx, &req)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, st.ErrorCode, commonpb.ErrorCode_SUCCESS)
|
||||
|
||||
coll_meta, err := svr.mt.GetCollectionByName(sch.Name)
|
||||
assert.Nil(t, err)
|
||||
t.Logf("collection id = %d", coll_meta.Id)
|
||||
assert.Equal(t, coll_meta.CreateTime, uint64(11))
|
||||
assert.Equal(t, coll_meta.Schema.Name, "col1")
|
||||
assert.Equal(t, coll_meta.Schema.AutoId, false)
|
||||
assert.Equal(t, len(coll_meta.Schema.Fields), 2)
|
||||
assert.Equal(t, coll_meta.Schema.Fields[0].Name, "col1_f1")
|
||||
assert.Equal(t, coll_meta.Schema.Fields[1].Name, "col1_f2")
|
||||
assert.Equal(t, coll_meta.Schema.Fields[0].DataType, schemapb.DataType_VECTOR_FLOAT)
|
||||
assert.Equal(t, coll_meta.Schema.Fields[1].DataType, schemapb.DataType_VECTOR_BINARY)
|
||||
assert.Equal(t, len(coll_meta.Schema.Fields[0].TypeParams), 2)
|
||||
assert.Equal(t, len(coll_meta.Schema.Fields[0].IndexParams), 2)
|
||||
assert.Equal(t, len(coll_meta.Schema.Fields[1].TypeParams), 2)
|
||||
assert.Equal(t, len(coll_meta.Schema.Fields[1].IndexParams), 2)
|
||||
assert.Equal(t, coll_meta.Schema.Fields[0].TypeParams[0].Key, "col1_f1_tk1")
|
||||
assert.Equal(t, coll_meta.Schema.Fields[0].TypeParams[1].Key, "col1_f1_tk2")
|
||||
assert.Equal(t, coll_meta.Schema.Fields[0].TypeParams[0].Value, "col1_f1_tv1")
|
||||
assert.Equal(t, coll_meta.Schema.Fields[0].TypeParams[1].Value, "col1_f1_tv2")
|
||||
assert.Equal(t, coll_meta.Schema.Fields[0].IndexParams[0].Key, "col1_f1_ik1")
|
||||
assert.Equal(t, coll_meta.Schema.Fields[0].IndexParams[1].Key, "col1_f1_ik2")
|
||||
assert.Equal(t, coll_meta.Schema.Fields[0].IndexParams[0].Value, "col1_f1_iv1")
|
||||
assert.Equal(t, coll_meta.Schema.Fields[0].IndexParams[1].Value, "col1_f1_iv2")
|
||||
|
||||
assert.Equal(t, coll_meta.Schema.Fields[1].TypeParams[0].Key, "col1_f2_tk1")
|
||||
assert.Equal(t, coll_meta.Schema.Fields[1].TypeParams[1].Key, "col1_f2_tk2")
|
||||
assert.Equal(t, coll_meta.Schema.Fields[1].TypeParams[0].Value, "col1_f2_tv1")
|
||||
assert.Equal(t, coll_meta.Schema.Fields[1].TypeParams[1].Value, "col1_f2_tv2")
|
||||
assert.Equal(t, coll_meta.Schema.Fields[1].IndexParams[0].Key, "col1_f2_ik1")
|
||||
assert.Equal(t, coll_meta.Schema.Fields[1].IndexParams[1].Key, "col1_f2_ik2")
|
||||
assert.Equal(t, coll_meta.Schema.Fields[1].IndexParams[0].Value, "col1_f2_iv1")
|
||||
assert.Equal(t, coll_meta.Schema.Fields[1].IndexParams[1].Value, "col1_f2_iv2")
|
||||
|
||||
svr.Close()
|
||||
}
|
|
@ -1,12 +1,13 @@
|
|||
|
||||
package id
|
||||
|
||||
import (
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv"
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/tso"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/tsoutil"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
|
||||
)
|
||||
|
||||
|
||||
type UniqueID = typeutil.UniqueID
|
||||
|
||||
// GlobalTSOAllocator is the global single point TSO allocator.
|
||||
|
@ -16,18 +17,13 @@ type GlobalIdAllocator struct {
|
|||
|
||||
var allocator *GlobalIdAllocator
|
||||
|
||||
func Init() {
|
||||
InitGlobalIdAllocator("idTimestamp", tsoutil.NewTSOKVBase("gid"))
|
||||
}
|
||||
|
||||
func InitGlobalIdAllocator(key string, base kv.KVBase) {
|
||||
func InitGlobalIdAllocator(key string, base kv.KVBase){
|
||||
allocator = NewGlobalIdAllocator(key, base)
|
||||
allocator.Initialize()
|
||||
}
|
||||
|
||||
func NewGlobalIdAllocator(key string, base kv.KVBase) *GlobalIdAllocator {
|
||||
func NewGlobalIdAllocator(key string, base kv.KVBase) * GlobalIdAllocator{
|
||||
return &GlobalIdAllocator{
|
||||
allocator: tso.NewGlobalTSOAllocator(key, base),
|
||||
allocator: tso.NewGlobalTSOAllocator( key, base),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -1,19 +1,17 @@
|
|||
package id
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/tsoutil"
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv/mockkv"
|
||||
"os"
|
||||
|
||||
"testing"
|
||||
)
|
||||
|
||||
var GIdAllocator *GlobalIdAllocator
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
conf.LoadConfig("config.yaml")
|
||||
GIdAllocator = NewGlobalIdAllocator("idTimestamp", tsoutil.NewTSOKVBase("gid"))
|
||||
GIdAllocator = NewGlobalIdAllocator("idTimestamp", mockkv.NewEtcdKV())
|
||||
exitCode := m.Run()
|
||||
os.Exit(exitCode)
|
||||
}
|
||||
|
@ -32,8 +30,8 @@ func TestGlobalIdAllocator_AllocOne(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestGlobalIdAllocator_Alloc(t *testing.T) {
|
||||
count := uint32(2 << 10)
|
||||
count := uint32(2<<10)
|
||||
idStart, idEnd, err := GIdAllocator.Alloc(count)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, count, uint32(idEnd-idStart))
|
||||
}
|
||||
assert.Equal(t, count, uint32(idEnd - idStart))
|
||||
}
|
|
@ -3,20 +3,18 @@ package master
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/zilliztech/milvus-distributed/internal/errors"
|
||||
"log"
|
||||
"math/rand"
|
||||
"net"
|
||||
"path"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/id"
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/tso"
|
||||
|
||||
"github.com/apache/pulsar-client-go/pulsar"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/id"
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv"
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/controller"
|
||||
|
@ -25,6 +23,7 @@ import (
|
|||
"github.com/zilliztech/milvus-distributed/internal/proto/masterpb"
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/tso"
|
||||
"go.etcd.io/etcd/clientv3"
|
||||
)
|
||||
|
||||
|
@ -44,6 +43,9 @@ type Master struct {
|
|||
//grpc server
|
||||
grpcServer *grpc.Server
|
||||
|
||||
// for tso.
|
||||
tsoAllocator tso.Allocator
|
||||
|
||||
// pulsar client
|
||||
pc *informer.PulsarClient
|
||||
|
||||
|
@ -52,50 +54,46 @@ type Master struct {
|
|||
|
||||
kvBase *kv.EtcdKV
|
||||
scheduler *ddRequestScheduler
|
||||
mt *metaTable
|
||||
mt metaTable
|
||||
// Add callback functions at different stages
|
||||
startCallbacks []func()
|
||||
closeCallbacks []func()
|
||||
}
|
||||
|
||||
func newKVBase(kv_root string, etcdAddr []string) *kv.EtcdKV {
|
||||
cli, _ := clientv3.New(clientv3.Config{
|
||||
Endpoints: etcdAddr,
|
||||
func newTSOKVBase(subPath string) * kv.EtcdKV{
|
||||
etcdAddr := conf.Config.Etcd.Address
|
||||
etcdAddr += ":"
|
||||
etcdAddr += strconv.FormatInt(int64(conf.Config.Etcd.Port), 10)
|
||||
client, _ := clientv3.New(clientv3.Config{
|
||||
Endpoints: []string{etcdAddr},
|
||||
DialTimeout: 5 * time.Second,
|
||||
})
|
||||
kvBase := kv.NewEtcdKV(cli, kv_root)
|
||||
return kv.NewEtcdKV(client, path.Join(conf.Config.Etcd.Rootpath, subPath))
|
||||
}
|
||||
|
||||
func newKVBase() *kv.EtcdKV {
|
||||
etcdAddr := conf.Config.Etcd.Address
|
||||
etcdAddr += ":"
|
||||
etcdAddr += strconv.FormatInt(int64(conf.Config.Etcd.Port), 10)
|
||||
cli, _ := clientv3.New(clientv3.Config{
|
||||
Endpoints: []string{etcdAddr},
|
||||
DialTimeout: 5 * time.Second,
|
||||
})
|
||||
kvBase := kv.NewEtcdKV(cli, conf.Config.Etcd.Rootpath)
|
||||
return kvBase
|
||||
}
|
||||
|
||||
func Init() {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
id.Init()
|
||||
tso.Init()
|
||||
}
|
||||
|
||||
// CreateServer creates the UNINITIALIZED pd server with given configuration.
|
||||
func CreateServer(ctx context.Context, kv_root_path string, meta_root_path, tso_root_path string, etcdAddr []string) (*Master, error) {
|
||||
func CreateServer(ctx context.Context) (*Master, error) {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
Init()
|
||||
|
||||
etcdClient, err := clientv3.New(clientv3.Config{Endpoints: etcdAddr})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
etcdkv := kv.NewEtcdKV(etcdClient, meta_root_path)
|
||||
metakv, err := NewMetaTable(etcdkv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
id.InitGlobalIdAllocator("idTimestamp", newTSOKVBase("gid"))
|
||||
m := &Master{
|
||||
ctx: ctx,
|
||||
startTimestamp: time.Now().Unix(),
|
||||
kvBase: newKVBase(kv_root_path, etcdAddr),
|
||||
scheduler: NewDDRequestScheduler(),
|
||||
mt: metakv,
|
||||
kvBase: newKVBase(),
|
||||
ssChan: make(chan internalpb.SegmentStatistics, 10),
|
||||
pc: informer.NewPulsarClient(),
|
||||
tsoAllocator: tso.NewGlobalTSOAllocator("timestamp", newTSOKVBase("tso")),
|
||||
}
|
||||
m.grpcServer = grpc.NewServer()
|
||||
masterpb.RegisterMasterServer(m.grpcServer, m)
|
||||
|
@ -153,13 +151,13 @@ func (s *Master) IsClosed() bool {
|
|||
}
|
||||
|
||||
// Run runs the pd server.
|
||||
func (s *Master) Run(grpcPort int64) error {
|
||||
func (s *Master) Run() error {
|
||||
|
||||
if err := s.startServer(s.ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.startServerLoop(s.ctx, grpcPort)
|
||||
s.startServerLoop(s.ctx)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
@ -174,28 +172,18 @@ func (s *Master) LoopContext() context.Context {
|
|||
return s.serverLoopCtx
|
||||
}
|
||||
|
||||
func (s *Master) startServerLoop(ctx context.Context, grpcPort int64) {
|
||||
func (s *Master) startServerLoop(ctx context.Context) {
|
||||
s.serverLoopCtx, s.serverLoopCancel = context.WithCancel(ctx)
|
||||
s.serverLoopWg.Add(3)
|
||||
//go s.Se
|
||||
|
||||
s.serverLoopWg.Add(1)
|
||||
go s.grpcLoop(grpcPort)
|
||||
|
||||
//s.serverLoopWg.Add(1)
|
||||
//go s.pulsarLoop()
|
||||
|
||||
s.serverLoopWg.Add(1)
|
||||
go s.tasksExecutionLoop()
|
||||
|
||||
s.serverLoopWg.Add(1)
|
||||
go s.grpcLoop()
|
||||
go s.pulsarLoop()
|
||||
go s.segmentStatisticsLoop()
|
||||
|
||||
}
|
||||
|
||||
func (s *Master) stopServerLoop() {
|
||||
if s.grpcServer != nil {
|
||||
if s.grpcServer != nil{
|
||||
s.grpcServer.GracefulStop()
|
||||
log.Printf("server is cloded, exit grpc server")
|
||||
}
|
||||
s.serverLoopCancel()
|
||||
s.serverLoopWg.Wait()
|
||||
|
@ -206,11 +194,11 @@ func (s *Master) StartTimestamp() int64 {
|
|||
return s.startTimestamp
|
||||
}
|
||||
|
||||
func (s *Master) grpcLoop(grpcPort int64) {
|
||||
func (s *Master) grpcLoop() {
|
||||
defer s.serverLoopWg.Done()
|
||||
|
||||
defaultGRPCPort := ":"
|
||||
defaultGRPCPort += strconv.FormatInt(grpcPort, 10)
|
||||
defaultGRPCPort += strconv.FormatInt(int64(conf.Config.Master.Port), 10)
|
||||
lis, err := net.Listen("tcp", defaultGRPCPort)
|
||||
if err != nil {
|
||||
log.Printf("failed to listen: %v", err)
|
||||
|
@ -257,7 +245,7 @@ func (s *Master) pulsarLoop() {
|
|||
s.ssChan <- m
|
||||
consumer.Ack(msg)
|
||||
case <-ctx.Done():
|
||||
log.Print("server is closed, exit pulsar loop")
|
||||
log.Print("server is closed, exit etcd leader loop")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
@ -270,16 +258,18 @@ func (s *Master) tasksExecutionLoop() {
|
|||
for {
|
||||
select {
|
||||
case task := <-s.scheduler.reqQueue:
|
||||
timeStamp, err := (task).Ts()
|
||||
timeStamp, err := (*task).Ts()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
} else {
|
||||
if timeStamp < s.scheduler.scheduleTimeStamp {
|
||||
task.Notify(errors.Errorf("input timestamp = %d, schduler timestamp = %d", timeStamp, s.scheduler.scheduleTimeStamp))
|
||||
_ = (*task).NotifyTimeout()
|
||||
} else {
|
||||
s.scheduler.scheduleTimeStamp = timeStamp
|
||||
err = task.Execute()
|
||||
task.Notify(err)
|
||||
err := (*task).Execute()
|
||||
if err != nil {
|
||||
log.Println("request execution failed caused by error:", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
case <-ctx.Done():
|
||||
|
@ -300,7 +290,7 @@ func (s *Master) segmentStatisticsLoop() {
|
|||
case ss := <-s.ssChan:
|
||||
controller.ComputeCloseTime(ss, s.kvBase)
|
||||
case <-ctx.Done():
|
||||
log.Print("server is closed, exit segment statistics loop")
|
||||
log.Print("server is closed, exit etcd leader loop")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
|
@ -59,6 +59,7 @@ func (t *createPartitionTask) Ts() (Timestamp, error) {
|
|||
|
||||
func (t *createPartitionTask) Execute() error {
|
||||
if t.req == nil {
|
||||
_ = t.Notify()
|
||||
return errors.New("null request")
|
||||
}
|
||||
|
||||
|
@ -66,6 +67,7 @@ func (t *createPartitionTask) Execute() error {
|
|||
collectionName := partitionName.CollectionName
|
||||
collectionMeta, err := t.mt.GetCollectionByName(collectionName)
|
||||
if err != nil {
|
||||
_ = t.Notify()
|
||||
return err
|
||||
}
|
||||
|
||||
|
@ -73,15 +75,18 @@ func (t *createPartitionTask) Execute() error {
|
|||
|
||||
collectionJson, err := json.Marshal(&collectionMeta)
|
||||
if err != nil {
|
||||
return err
|
||||
_ = t.Notify()
|
||||
return errors.New("marshal collection failed")
|
||||
}
|
||||
|
||||
collectionId := collectionMeta.Id
|
||||
err = (*t.kvBase).Save(partitionMetaPrefix+strconv.FormatInt(collectionId, 10), string(collectionJson))
|
||||
if err != nil {
|
||||
return err
|
||||
_ = t.Notify()
|
||||
return errors.New("save collection failed")
|
||||
}
|
||||
|
||||
_ = t.Notify()
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -103,6 +108,7 @@ func (t *dropPartitionTask) Ts() (Timestamp, error) {
|
|||
|
||||
func (t *dropPartitionTask) Execute() error {
|
||||
if t.req == nil {
|
||||
_ = t.Notify()
|
||||
return errors.New("null request")
|
||||
}
|
||||
|
||||
|
@ -110,6 +116,7 @@ func (t *dropPartitionTask) Execute() error {
|
|||
collectionName := partitionName.CollectionName
|
||||
collectionMeta, err := t.mt.GetCollectionByName(collectionName)
|
||||
if err != nil {
|
||||
_ = t.Notify()
|
||||
return err
|
||||
}
|
||||
|
||||
|
@ -120,15 +127,18 @@ func (t *dropPartitionTask) Execute() error {
|
|||
|
||||
collectionJson, err := json.Marshal(&collectionMeta)
|
||||
if err != nil {
|
||||
return err
|
||||
_ = t.Notify()
|
||||
return errors.New("marshal collection failed")
|
||||
}
|
||||
|
||||
collectionId := collectionMeta.Id
|
||||
err = (*t.kvBase).Save(partitionMetaPrefix+strconv.FormatInt(collectionId, 10), string(collectionJson))
|
||||
if err != nil {
|
||||
return err
|
||||
_ = t.Notify()
|
||||
return errors.New("save collection failed")
|
||||
}
|
||||
|
||||
_ = t.Notify()
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -150,6 +160,7 @@ func (t *hasPartitionTask) Ts() (Timestamp, error) {
|
|||
|
||||
func (t *hasPartitionTask) Execute() error {
|
||||
if t.req == nil {
|
||||
_ = t.Notify()
|
||||
return errors.New("null request")
|
||||
}
|
||||
|
||||
|
@ -162,6 +173,7 @@ func (t *hasPartitionTask) Execute() error {
|
|||
|
||||
t.hasPartition = t.mt.HasPartition(collectionMeta.Id, partitionName.Tag)
|
||||
|
||||
_ = t.Notify()
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -183,6 +195,7 @@ func (t *describePartitionTask) Ts() (Timestamp, error) {
|
|||
|
||||
func (t *describePartitionTask) Execute() error {
|
||||
if t.req == nil {
|
||||
_ = t.Notify()
|
||||
return errors.New("null request")
|
||||
}
|
||||
|
||||
|
@ -197,6 +210,7 @@ func (t *describePartitionTask) Execute() error {
|
|||
|
||||
t.description = &description
|
||||
|
||||
_ = t.Notify()
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -218,6 +232,7 @@ func (t *showPartitionTask) Ts() (Timestamp, error) {
|
|||
|
||||
func (t *showPartitionTask) Execute() error {
|
||||
if t.req == nil {
|
||||
_ = t.Notify()
|
||||
return errors.New("null request")
|
||||
}
|
||||
|
||||
|
@ -237,5 +252,6 @@ func (t *showPartitionTask) Execute() error {
|
|||
|
||||
t.stringListResponse = &stringListResponse
|
||||
|
||||
_ = t.Notify()
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
package master
|
||||
|
||||
import "math/rand"
|
||||
|
||||
type ddRequestScheduler struct {
|
||||
reqQueue chan task
|
||||
reqQueue chan *task
|
||||
scheduleTimeStamp Timestamp
|
||||
}
|
||||
|
||||
|
@ -11,17 +9,12 @@ func NewDDRequestScheduler() *ddRequestScheduler {
|
|||
const channelSize = 1024
|
||||
|
||||
rs := ddRequestScheduler{
|
||||
reqQueue: make(chan task, channelSize),
|
||||
reqQueue: make(chan *task, channelSize),
|
||||
}
|
||||
return &rs
|
||||
}
|
||||
|
||||
func (rs *ddRequestScheduler) Enqueue(task task) error {
|
||||
func (rs *ddRequestScheduler) Enqueue(task *task) error {
|
||||
rs.reqQueue <- task
|
||||
return nil
|
||||
}
|
||||
|
||||
//TODO, allocGlobalId
|
||||
func allocGlobalId() (UniqueID, error) {
|
||||
return rand.Int63(), nil
|
||||
}
|
||||
|
|
|
@ -13,7 +13,7 @@ import (
|
|||
type baseTask struct {
|
||||
kvBase *kv.EtcdKV
|
||||
mt *metaTable
|
||||
cv chan error
|
||||
cv chan int
|
||||
}
|
||||
|
||||
type task interface {
|
||||
|
@ -21,23 +21,27 @@ type task interface {
|
|||
Ts() (Timestamp, error)
|
||||
Execute() error
|
||||
WaitToFinish(ctx context.Context) error
|
||||
Notify(err error)
|
||||
Notify() error
|
||||
NotifyTimeout() error
|
||||
}
|
||||
|
||||
func (bt *baseTask) Notify(err error) {
|
||||
bt.cv <- err
|
||||
func (bt *baseTask) Notify() error {
|
||||
bt.cv <- 0
|
||||
return nil
|
||||
}
|
||||
|
||||
func (bt *baseTask) NotifyTimeout() error {
|
||||
bt.cv <- 0
|
||||
return errors.New("request timeout")
|
||||
}
|
||||
|
||||
func (bt *baseTask) WaitToFinish(ctx context.Context) error {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return errors.Errorf("context done")
|
||||
case err, ok := <-bt.cv:
|
||||
if !ok {
|
||||
return errors.Errorf("notify chan closed")
|
||||
}
|
||||
return err
|
||||
return nil
|
||||
case <-bt.cv:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -3,261 +3,213 @@ package timesync
|
|||
import (
|
||||
"context"
|
||||
"log"
|
||||
"math"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/errors"
|
||||
ms "github.com/zilliztech/milvus-distributed/internal/msgstream"
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
|
||||
"github.com/apache/pulsar-client-go/pulsar"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
)
|
||||
|
||||
type (
|
||||
softTimeTickBarrier struct {
|
||||
peer2LastTt map[UniqueID]Timestamp
|
||||
minTtInterval Timestamp
|
||||
lastTt Timestamp
|
||||
outTt chan Timestamp
|
||||
ttStream ms.MsgStream
|
||||
ctx context.Context
|
||||
closeCh chan struct{} // close goroutinue in Start()
|
||||
closed bool
|
||||
const stopReadFlagId int64 = -1
|
||||
|
||||
type TimeTickReader struct {
|
||||
pulsarClient pulsar.Client
|
||||
|
||||
timeTickConsumer pulsar.Consumer
|
||||
readerProducer []pulsar.Producer
|
||||
|
||||
interval int64
|
||||
proxyIdList []UniqueID
|
||||
|
||||
timeTickPeerProxy map[UniqueID]Timestamp
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func (r *TimeTickReader) Start() {
|
||||
go r.readTimeTick()
|
||||
go r.timeSync()
|
||||
|
||||
}
|
||||
|
||||
func (r *TimeTickReader) Close() {
|
||||
if r.timeTickConsumer != nil {
|
||||
r.timeTickConsumer.Close()
|
||||
}
|
||||
|
||||
hardTimeTickBarrier struct {
|
||||
peer2Tt map[UniqueID]Timestamp
|
||||
outTt chan Timestamp
|
||||
ttStream ms.MsgStream
|
||||
ctx context.Context
|
||||
closeCh chan struct{} // close goroutinue in Start()
|
||||
closed bool
|
||||
for i := 0; i < len(r.readerProducer); i++ {
|
||||
if r.readerProducer[i] != nil {
|
||||
r.readerProducer[i].Close()
|
||||
}
|
||||
}
|
||||
)
|
||||
if r.pulsarClient != nil {
|
||||
r.pulsarClient.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func (ttBarrier *softTimeTickBarrier) GetTimeTick() (Timestamp, error) {
|
||||
isEmpty := true
|
||||
func (r *TimeTickReader) timeSync() {
|
||||
ctx := r.ctx
|
||||
for {
|
||||
|
||||
if ttBarrier.closed {
|
||||
return 0, errors.Errorf("[GetTimeTick] closed.")
|
||||
}
|
||||
|
||||
select {
|
||||
case ts := <-ttBarrier.outTt:
|
||||
isEmpty = false
|
||||
ttBarrier.lastTt = ts
|
||||
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
if isEmpty {
|
||||
continue
|
||||
time.Sleep(time.Millisecond * time.Duration(r.interval))
|
||||
var minTimeStamp Timestamp
|
||||
for _, minTimeStamp = range r.timeTickPeerProxy {
|
||||
break
|
||||
}
|
||||
return ttBarrier.lastTt, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (ttBarrier *softTimeTickBarrier) Start() error {
|
||||
ttBarrier.closeCh = make(chan struct{})
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
|
||||
case <-ttBarrier.closeCh:
|
||||
log.Printf("[TtBarrierStart] closed\n")
|
||||
return
|
||||
|
||||
case <-ttBarrier.ctx.Done():
|
||||
log.Printf("[TtBarrierStart] %s\n", ttBarrier.ctx.Err())
|
||||
ttBarrier.closed = true
|
||||
return
|
||||
|
||||
case ttmsgs := <-ttBarrier.ttStream.Chan():
|
||||
if len(ttmsgs.Msgs) > 0 {
|
||||
for _, timetickmsg := range ttmsgs.Msgs {
|
||||
ttmsg := (*timetickmsg).(*ms.TimeTickMsg)
|
||||
oldT, ok := ttBarrier.peer2LastTt[ttmsg.PeerId]
|
||||
log.Printf("[softTimeTickBarrier] peer(%d)=%d\n", ttmsg.PeerId, ttmsg.Timestamp)
|
||||
|
||||
if !ok {
|
||||
log.Printf("[softTimeTickBarrier] Warning: peerId %d not exist\n", ttmsg.PeerId)
|
||||
continue
|
||||
}
|
||||
|
||||
if ttmsg.Timestamp > oldT {
|
||||
ttBarrier.peer2LastTt[ttmsg.PeerId] = ttmsg.Timestamp
|
||||
|
||||
// get a legal Timestamp
|
||||
ts := ttBarrier.minTimestamp()
|
||||
|
||||
if ttBarrier.lastTt != 0 && ttBarrier.minTtInterval > ts-ttBarrier.lastTt {
|
||||
continue
|
||||
}
|
||||
|
||||
ttBarrier.outTt <- ts
|
||||
}
|
||||
}
|
||||
for _, ts := range r.timeTickPeerProxy {
|
||||
if ts < minTimeStamp {
|
||||
minTimeStamp = ts
|
||||
}
|
||||
|
||||
default:
|
||||
}
|
||||
//send timestamp flag to reader channel
|
||||
msg := internalpb.TimeTickMsg{
|
||||
Timestamp: minTimeStamp,
|
||||
MsgType: internalpb.MsgType_kTimeTick,
|
||||
}
|
||||
payload, err := proto.Marshal(&msg)
|
||||
if err != nil {
|
||||
//TODO log error
|
||||
log.Printf("Marshal InsertOrDeleteMsg flag error %v", err)
|
||||
} else {
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(len(r.readerProducer))
|
||||
for index := range r.readerProducer {
|
||||
go r.sendEOFMsg(ctx, &pulsar.ProducerMessage{Payload: payload}, index, &wg)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func NewSoftTimeTickBarrier(ctx context.Context,
|
||||
ttStream *ms.MsgStream,
|
||||
peerIds []UniqueID,
|
||||
minTtInterval Timestamp) *softTimeTickBarrier {
|
||||
|
||||
if len(peerIds) <= 0 {
|
||||
log.Printf("[NewSoftTimeTickBarrier] Error: peerIds is emtpy!\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
sttbarrier := softTimeTickBarrier{}
|
||||
sttbarrier.minTtInterval = minTtInterval
|
||||
sttbarrier.ttStream = *ttStream
|
||||
sttbarrier.outTt = make(chan Timestamp, 1024)
|
||||
sttbarrier.ctx = ctx
|
||||
sttbarrier.closed = false
|
||||
|
||||
sttbarrier.peer2LastTt = make(map[UniqueID]Timestamp)
|
||||
for _, id := range peerIds {
|
||||
sttbarrier.peer2LastTt[id] = Timestamp(0)
|
||||
}
|
||||
if len(peerIds) != len(sttbarrier.peer2LastTt) {
|
||||
log.Printf("[NewSoftTimeTickBarrier] Warning: there are duplicate peerIds!\n")
|
||||
}
|
||||
|
||||
return &sttbarrier
|
||||
}
|
||||
|
||||
func (ttBarrier *softTimeTickBarrier) Close() {
|
||||
|
||||
if ttBarrier.closeCh != nil {
|
||||
ttBarrier.closeCh <- struct{}{}
|
||||
}
|
||||
|
||||
ttBarrier.closed = true
|
||||
}
|
||||
|
||||
func (ttBarrier *softTimeTickBarrier) minTimestamp() Timestamp {
|
||||
tempMin := Timestamp(math.MaxUint64)
|
||||
for _, tt := range ttBarrier.peer2LastTt {
|
||||
if tt < tempMin {
|
||||
tempMin = tt
|
||||
}
|
||||
}
|
||||
return tempMin
|
||||
}
|
||||
|
||||
func (ttBarrier *hardTimeTickBarrier) GetTimeTick() (Timestamp, error) {
|
||||
func (r *TimeTickReader) readTimeTick() {
|
||||
for {
|
||||
|
||||
if ttBarrier.closed {
|
||||
return 0, errors.Errorf("[GetTimeTick] closed.")
|
||||
}
|
||||
|
||||
select {
|
||||
case ts := <-ttBarrier.outTt:
|
||||
return ts, nil
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (ttBarrier *hardTimeTickBarrier) Start() error {
|
||||
ttBarrier.closeCh = make(chan struct{})
|
||||
|
||||
go func() {
|
||||
// Last timestamp synchronized
|
||||
state := Timestamp(0)
|
||||
for {
|
||||
select {
|
||||
|
||||
case <-ttBarrier.closeCh:
|
||||
log.Printf("[TtBarrierStart] closed\n")
|
||||
return
|
||||
|
||||
case <-ttBarrier.ctx.Done():
|
||||
log.Printf("[TtBarrierStart] %s\n", ttBarrier.ctx.Err())
|
||||
ttBarrier.closed = true
|
||||
return
|
||||
|
||||
case ttmsgs := <-ttBarrier.ttStream.Chan():
|
||||
if len(ttmsgs.Msgs) > 0 {
|
||||
for _, timetickmsg := range ttmsgs.Msgs {
|
||||
|
||||
// Suppose ttmsg.Timestamp from stream is always larger than the previous one,
|
||||
// that `ttmsg.Timestamp > oldT`
|
||||
ttmsg := (*timetickmsg).(*ms.TimeTickMsg)
|
||||
log.Printf("[hardTimeTickBarrier] peer(%d)=%d\n", ttmsg.PeerId, ttmsg.Timestamp)
|
||||
|
||||
oldT, ok := ttBarrier.peer2Tt[ttmsg.PeerId]
|
||||
if !ok {
|
||||
log.Printf("[hardTimeTickBarrier] Warning: peerId %d not exist\n", ttmsg.PeerId)
|
||||
continue
|
||||
}
|
||||
|
||||
if oldT > state {
|
||||
log.Printf("[hardTimeTickBarrier] Warning: peer(%d) timestamp(%d) ahead\n",
|
||||
ttmsg.PeerId, ttmsg.Timestamp)
|
||||
}
|
||||
|
||||
ttBarrier.peer2Tt[ttmsg.PeerId] = ttmsg.Timestamp
|
||||
|
||||
newState := ttBarrier.minTimestamp()
|
||||
if newState > state {
|
||||
ttBarrier.outTt <- newState
|
||||
state = newState
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
case <-r.ctx.Done():
|
||||
return
|
||||
case cm, ok := <-r.timeTickConsumer.Chan():
|
||||
if ok == false {
|
||||
log.Printf("timesync consumer closed")
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ttBarrier *hardTimeTickBarrier) minTimestamp() Timestamp {
|
||||
tempMin := Timestamp(math.MaxUint64)
|
||||
for _, tt := range ttBarrier.peer2Tt {
|
||||
if tt < tempMin {
|
||||
tempMin = tt
|
||||
msg := cm.Message
|
||||
var tsm internalpb.TimeTickMsg
|
||||
if err := proto.Unmarshal(msg.Payload(), &tsm); err != nil {
|
||||
log.Printf("UnMarshal timetick flag error %v", err)
|
||||
}
|
||||
|
||||
r.timeTickPeerProxy[tsm.PeerId] = tsm.Timestamp
|
||||
r.timeTickConsumer.AckID(msg.ID())
|
||||
}
|
||||
}
|
||||
return tempMin
|
||||
}
|
||||
|
||||
func NewHardTimeTickBarrier(ctx context.Context,
|
||||
ttStream *ms.MsgStream,
|
||||
peerIds []UniqueID) *hardTimeTickBarrier {
|
||||
|
||||
if len(peerIds) <= 0 {
|
||||
log.Printf("[NewSoftTimeTickBarrier] Error: peerIds is emtpy!")
|
||||
return nil
|
||||
func (r *TimeTickReader) sendEOFMsg(ctx context.Context, msg *pulsar.ProducerMessage, index int, wg *sync.WaitGroup) {
|
||||
if _, err := r.readerProducer[index].Send(ctx, msg); err != nil {
|
||||
log.Printf("Send timesync flag error %v", err)
|
||||
}
|
||||
|
||||
sttbarrier := hardTimeTickBarrier{}
|
||||
sttbarrier.ttStream = *ttStream
|
||||
sttbarrier.outTt = make(chan Timestamp, 1024)
|
||||
sttbarrier.ctx = ctx
|
||||
sttbarrier.closed = false
|
||||
|
||||
sttbarrier.peer2Tt = make(map[UniqueID]Timestamp)
|
||||
for _, id := range peerIds {
|
||||
sttbarrier.peer2Tt[id] = Timestamp(0)
|
||||
}
|
||||
if len(peerIds) != len(sttbarrier.peer2Tt) {
|
||||
log.Printf("[NewSoftTimeTickBarrier] Warning: there are duplicate peerIds!")
|
||||
}
|
||||
|
||||
return &sttbarrier
|
||||
wg.Done()
|
||||
}
|
||||
|
||||
func (ttBarrier *hardTimeTickBarrier) Close() {
|
||||
if ttBarrier.closeCh != nil {
|
||||
ttBarrier.closeCh <- struct{}{}
|
||||
func TimeTickService() {
|
||||
timeTickTopic := "timeTick"
|
||||
timeTickSubName := "master"
|
||||
readTopics := make([]string, 0)
|
||||
for i := conf.Config.Reader.TopicStart; i < conf.Config.Reader.TopicEnd; i++ {
|
||||
str := "InsertOrDelete-"
|
||||
str = str + strconv.Itoa(i)
|
||||
readTopics = append(readTopics, str)
|
||||
}
|
||||
ttBarrier.closed = true
|
||||
return
|
||||
|
||||
proxyIdList := conf.Config.Master.ProxyIdList
|
||||
timeTickReader := newTimeTickReader(context.Background(), timeTickTopic, timeTickSubName, readTopics, proxyIdList)
|
||||
timeTickReader.Start()
|
||||
}
|
||||
|
||||
func newTimeTickReader(
|
||||
ctx context.Context,
|
||||
timeTickTopic string,
|
||||
timeTickSubName string,
|
||||
readTopics []string,
|
||||
proxyIdList []UniqueID,
|
||||
) *TimeTickReader {
|
||||
pulsarAddr := "pulsar://"
|
||||
pulsarAddr += conf.Config.Pulsar.Address
|
||||
pulsarAddr += ":"
|
||||
pulsarAddr += strconv.FormatInt(int64(conf.Config.Pulsar.Port), 10)
|
||||
interval := int64(conf.Config.Timesync.Interval)
|
||||
|
||||
//check if proxyId has duplication
|
||||
if len(proxyIdList) == 0 {
|
||||
log.Printf("proxy id list is empty")
|
||||
}
|
||||
if len(proxyIdList) > 1 {
|
||||
sort.Slice(proxyIdList, func(i int, j int) bool { return proxyIdList[i] < proxyIdList[j] })
|
||||
}
|
||||
for i := 1; i < len(proxyIdList); i++ {
|
||||
if proxyIdList[i] == proxyIdList[i-1] {
|
||||
log.Printf("there are two proxies have the same id = %d", proxyIdList[i])
|
||||
}
|
||||
}
|
||||
r := TimeTickReader{}
|
||||
r.interval = interval
|
||||
r.proxyIdList = proxyIdList
|
||||
readerQueueSize := conf.Config.Reader.ReaderQueueSize
|
||||
|
||||
//check if read topic is empty
|
||||
if len(readTopics) == 0 {
|
||||
log.Printf("read topic is empyt")
|
||||
}
|
||||
//set default value
|
||||
if readerQueueSize == 0 {
|
||||
readerQueueSize = 1024
|
||||
}
|
||||
|
||||
r.timeTickPeerProxy = make(map[UniqueID]Timestamp)
|
||||
r.ctx = ctx
|
||||
|
||||
var client pulsar.Client
|
||||
var err error
|
||||
if conf.Config.Pulsar.Authentication {
|
||||
client, err = pulsar.NewClient(pulsar.ClientOptions{
|
||||
URL: pulsarAddr,
|
||||
Authentication: pulsar.NewAuthenticationToken(conf.Config.Pulsar.Token),
|
||||
})
|
||||
} else {
|
||||
client, err = pulsar.NewClient(pulsar.ClientOptions{URL: pulsarAddr})
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
log.Printf("connect pulsar failed, %v", err)
|
||||
}
|
||||
r.pulsarClient = client
|
||||
|
||||
timeSyncChan := make(chan pulsar.ConsumerMessage, len(r.proxyIdList))
|
||||
if r.timeTickConsumer, err = r.pulsarClient.Subscribe(pulsar.ConsumerOptions{
|
||||
Topic: timeTickTopic,
|
||||
SubscriptionName: timeTickSubName,
|
||||
Type: pulsar.KeyShared,
|
||||
SubscriptionInitialPosition: pulsar.SubscriptionPositionEarliest,
|
||||
MessageChannel: timeSyncChan,
|
||||
}); err != nil {
|
||||
log.Printf("failed to subscribe topic %s, error = %v", timeTickTopic, err)
|
||||
}
|
||||
|
||||
r.readerProducer = make([]pulsar.Producer, 0, len(readTopics))
|
||||
for i := 0; i < len(readTopics); i++ {
|
||||
rp, err := r.pulsarClient.CreateProducer(pulsar.ProducerOptions{Topic: readTopics[i]})
|
||||
if err != nil {
|
||||
log.Printf("failed to create reader producer %s, error = %v", readTopics[i], err)
|
||||
}
|
||||
r.readerProducer = append(r.readerProducer, rp)
|
||||
}
|
||||
|
||||
return &r
|
||||
}
|
||||
|
|
|
@ -1,426 +0,0 @@
|
|||
package timesync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
ms "github.com/zilliztech/milvus-distributed/internal/msgstream"
|
||||
internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
)
|
||||
|
||||
func getTtMsg(msgType internalPb.MsgType, peerId UniqueID, timeStamp uint64) *ms.TsMsg {
|
||||
var tsMsg ms.TsMsg
|
||||
baseMsg := ms.BaseMsg{
|
||||
HashValues: []int32{int32(peerId)},
|
||||
}
|
||||
timeTickResult := internalPb.TimeTickMsg{
|
||||
MsgType: internalPb.MsgType_kTimeTick,
|
||||
PeerId: peerId,
|
||||
Timestamp: timeStamp,
|
||||
}
|
||||
timeTickMsg := &ms.TimeTickMsg{
|
||||
BaseMsg: baseMsg,
|
||||
TimeTickMsg: timeTickResult,
|
||||
}
|
||||
tsMsg = timeTickMsg
|
||||
return &tsMsg
|
||||
}
|
||||
|
||||
func initPulsarStream(pulsarAddress string,
|
||||
producerChannels []string,
|
||||
consumerChannels []string,
|
||||
consumerSubName string) (*ms.MsgStream, *ms.MsgStream) {
|
||||
|
||||
// set input stream
|
||||
inputStream := ms.NewPulsarMsgStream(context.Background(), 100)
|
||||
inputStream.SetPulsarCient(pulsarAddress)
|
||||
inputStream.CreatePulsarProducers(producerChannels)
|
||||
var input ms.MsgStream = inputStream
|
||||
|
||||
// set output stream
|
||||
outputStream := ms.NewPulsarMsgStream(context.Background(), 100)
|
||||
outputStream.SetPulsarCient(pulsarAddress)
|
||||
unmarshalDispatcher := ms.NewUnmarshalDispatcher()
|
||||
outputStream.CreatePulsarConsumers(consumerChannels, consumerSubName, unmarshalDispatcher, 100)
|
||||
outputStream.Start()
|
||||
var output ms.MsgStream = outputStream
|
||||
|
||||
return &input, &output
|
||||
}
|
||||
|
||||
func getMsgPack(ttmsgs [][2]int) *ms.MsgPack {
|
||||
msgPack := ms.MsgPack{}
|
||||
for _, vi := range ttmsgs {
|
||||
msgPack.Msgs = append(msgPack.Msgs, getTtMsg(internalPb.MsgType_kTimeTick, UniqueID(vi[0]), Timestamp(vi[1])))
|
||||
}
|
||||
return &msgPack
|
||||
}
|
||||
|
||||
func getEmptyMsgPack() *ms.MsgPack {
|
||||
msgPack := ms.MsgPack{}
|
||||
return &msgPack
|
||||
}
|
||||
|
||||
func producer(channels []string, ttmsgs [][2]int) (*ms.MsgStream, *ms.MsgStream) {
|
||||
pulsarAddress := "pulsar://localhost:6650"
|
||||
consumerSubName := "subTimetick"
|
||||
producerChannels := channels
|
||||
consumerChannels := channels
|
||||
|
||||
inputStream, outputStream := initPulsarStream(pulsarAddress, producerChannels, consumerChannels, consumerSubName)
|
||||
|
||||
msgPackAddr := getMsgPack(ttmsgs)
|
||||
(*inputStream).Produce(msgPackAddr)
|
||||
return inputStream, outputStream
|
||||
}
|
||||
|
||||
func TestTt_NewSoftTtBarrier(t *testing.T) {
|
||||
channels := []string{"NewSoftTtBarrier"}
|
||||
ttmsgs := [][2]int{
|
||||
{1, 10},
|
||||
{2, 20},
|
||||
{3, 30},
|
||||
{4, 40},
|
||||
{1, 30},
|
||||
{2, 30},
|
||||
}
|
||||
|
||||
inStream, ttStream := producer(channels, ttmsgs)
|
||||
defer func() {
|
||||
(*inStream).Close()
|
||||
(*ttStream).Close()
|
||||
}()
|
||||
|
||||
minTtInterval := Timestamp(10)
|
||||
|
||||
validPeerIds := []UniqueID{1, 2, 3}
|
||||
|
||||
sttbarrier := NewSoftTimeTickBarrier(context.TODO(), ttStream, validPeerIds, minTtInterval)
|
||||
assert.NotNil(t, sttbarrier)
|
||||
sttbarrier.Close()
|
||||
|
||||
validPeerIds2 := []UniqueID{1, 1, 1}
|
||||
sttbarrier = NewSoftTimeTickBarrier(context.TODO(), ttStream, validPeerIds2, minTtInterval)
|
||||
assert.NotNil(t, sttbarrier)
|
||||
sttbarrier.Close()
|
||||
|
||||
// invalid peerIds
|
||||
invalidPeerIds1 := make([]UniqueID, 0, 3)
|
||||
sttbarrier = NewSoftTimeTickBarrier(context.TODO(), ttStream, invalidPeerIds1, minTtInterval)
|
||||
assert.Nil(t, sttbarrier)
|
||||
|
||||
invalidPeerIds2 := []UniqueID{}
|
||||
sttbarrier = NewSoftTimeTickBarrier(context.TODO(), ttStream, invalidPeerIds2, minTtInterval)
|
||||
assert.Nil(t, sttbarrier)
|
||||
}
|
||||
|
||||
func TestTt_NewHardTtBarrier(t *testing.T) {
|
||||
channels := []string{"NewHardTtBarrier"}
|
||||
ttmsgs := [][2]int{
|
||||
{1, 10},
|
||||
{2, 20},
|
||||
{3, 30},
|
||||
{4, 40},
|
||||
{1, 30},
|
||||
{2, 30},
|
||||
}
|
||||
inStream, ttStream := producer(channels, ttmsgs)
|
||||
defer func() {
|
||||
(*inStream).Close()
|
||||
(*ttStream).Close()
|
||||
}()
|
||||
|
||||
validPeerIds := []UniqueID{1, 2, 3}
|
||||
|
||||
sttbarrier := NewHardTimeTickBarrier(context.TODO(), ttStream, validPeerIds)
|
||||
assert.NotNil(t, sttbarrier)
|
||||
sttbarrier.Close()
|
||||
|
||||
validPeerIds2 := []UniqueID{1, 1, 1}
|
||||
sttbarrier = NewHardTimeTickBarrier(context.TODO(), ttStream, validPeerIds2)
|
||||
assert.NotNil(t, sttbarrier)
|
||||
sttbarrier.Close()
|
||||
|
||||
// invalid peerIds
|
||||
invalidPeerIds1 := make([]UniqueID, 0, 3)
|
||||
sttbarrier = NewHardTimeTickBarrier(context.TODO(), ttStream, invalidPeerIds1)
|
||||
assert.Nil(t, sttbarrier)
|
||||
|
||||
invalidPeerIds2 := []UniqueID{}
|
||||
sttbarrier = NewHardTimeTickBarrier(context.TODO(), ttStream, invalidPeerIds2)
|
||||
assert.Nil(t, sttbarrier)
|
||||
}
|
||||
|
||||
func TestTt_SoftTtBarrierStart(t *testing.T) {
|
||||
channels := []string{"SoftTtBarrierStart"}
|
||||
|
||||
ttmsgs := [][2]int{
|
||||
{1, 10},
|
||||
{2, 20},
|
||||
{3, 30},
|
||||
{4, 40},
|
||||
{1, 30},
|
||||
{2, 30},
|
||||
}
|
||||
inStream, ttStream := producer(channels, ttmsgs)
|
||||
defer func() {
|
||||
(*inStream).Close()
|
||||
(*ttStream).Close()
|
||||
}()
|
||||
|
||||
minTtInterval := Timestamp(10)
|
||||
peerIds := []UniqueID{1, 2, 3}
|
||||
sttbarrier := NewSoftTimeTickBarrier(context.TODO(), ttStream, peerIds, minTtInterval)
|
||||
require.NotNil(t, sttbarrier)
|
||||
|
||||
sttbarrier.Start()
|
||||
defer sttbarrier.Close()
|
||||
|
||||
// Make sure all msgs in outputStream is consumed
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
ts, err := sttbarrier.GetTimeTick()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, Timestamp(30), ts)
|
||||
}
|
||||
|
||||
func TestTt_SoftTtBarrierGetTimeTickClose(t *testing.T) {
|
||||
channels := []string{"SoftTtBarrierGetTimeTickClose"}
|
||||
ttmsgs := [][2]int{
|
||||
{1, 10},
|
||||
{2, 20},
|
||||
{3, 30},
|
||||
{4, 40},
|
||||
{1, 30},
|
||||
{2, 30},
|
||||
}
|
||||
inStream, ttStream := producer(channels, ttmsgs)
|
||||
defer func() {
|
||||
(*inStream).Close()
|
||||
(*ttStream).Close()
|
||||
}()
|
||||
|
||||
minTtInterval := Timestamp(10)
|
||||
validPeerIds := []UniqueID{1, 2, 3}
|
||||
|
||||
sttbarrier := NewSoftTimeTickBarrier(context.TODO(), ttStream, validPeerIds, minTtInterval)
|
||||
require.NotNil(t, sttbarrier)
|
||||
|
||||
sttbarrier.Start()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
sttbarrier.Close()
|
||||
}()
|
||||
wg.Wait()
|
||||
|
||||
ts, err := sttbarrier.GetTimeTick()
|
||||
assert.NotNil(t, err)
|
||||
assert.Equal(t, Timestamp(0), ts)
|
||||
|
||||
// Receive empty msgPacks
|
||||
channels01 := []string{"GetTimeTick01"}
|
||||
ttmsgs01 := [][2]int{}
|
||||
inStream01, ttStream01 := producer(channels01, ttmsgs01)
|
||||
defer func() {
|
||||
(*inStream01).Close()
|
||||
(*ttStream01).Close()
|
||||
}()
|
||||
|
||||
minTtInterval = Timestamp(10)
|
||||
validPeerIds = []UniqueID{1, 2, 3}
|
||||
|
||||
sttbarrier01 := NewSoftTimeTickBarrier(context.TODO(), ttStream01, validPeerIds, minTtInterval)
|
||||
require.NotNil(t, sttbarrier01)
|
||||
sttbarrier01.Start()
|
||||
|
||||
var wg1 sync.WaitGroup
|
||||
wg1.Add(1)
|
||||
|
||||
go func() {
|
||||
defer wg1.Done()
|
||||
sttbarrier01.Close()
|
||||
}()
|
||||
|
||||
ts, err = sttbarrier01.GetTimeTick()
|
||||
assert.NotNil(t, err)
|
||||
assert.Equal(t, Timestamp(0), ts)
|
||||
}
|
||||
|
||||
func TestTt_SoftTtBarrierGetTimeTickCancel(t *testing.T) {
|
||||
channels := []string{"SoftTtBarrierGetTimeTickCancel"}
|
||||
ttmsgs := [][2]int{
|
||||
{1, 10},
|
||||
{2, 20},
|
||||
{3, 30},
|
||||
{4, 40},
|
||||
{1, 30},
|
||||
{2, 30},
|
||||
}
|
||||
inStream, ttStream := producer(channels, ttmsgs)
|
||||
defer func() {
|
||||
(*inStream).Close()
|
||||
(*ttStream).Close()
|
||||
}()
|
||||
|
||||
minTtInterval := Timestamp(10)
|
||||
validPeerIds := []UniqueID{1, 2, 3}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
sttbarrier := NewSoftTimeTickBarrier(ctx, ttStream, validPeerIds, minTtInterval)
|
||||
require.NotNil(t, sttbarrier)
|
||||
|
||||
sttbarrier.Start()
|
||||
|
||||
go func() {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
cancel()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
sttbarrier.Close()
|
||||
}()
|
||||
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
|
||||
ts, err := sttbarrier.GetTimeTick()
|
||||
assert.NotNil(t, err)
|
||||
assert.Equal(t, Timestamp(0), ts)
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
func TestTt_HardTtBarrierStart(t *testing.T) {
|
||||
channels := []string{"HardTtBarrierStart"}
|
||||
|
||||
ttmsgs := [][2]int{
|
||||
{1, 10},
|
||||
{2, 10},
|
||||
{3, 10},
|
||||
}
|
||||
|
||||
inStream, ttStream := producer(channels, ttmsgs)
|
||||
defer func() {
|
||||
(*inStream).Close()
|
||||
(*ttStream).Close()
|
||||
}()
|
||||
|
||||
peerIds := []UniqueID{1, 2, 3}
|
||||
sttbarrier := NewHardTimeTickBarrier(context.TODO(), ttStream, peerIds)
|
||||
require.NotNil(t, sttbarrier)
|
||||
|
||||
sttbarrier.Start()
|
||||
defer sttbarrier.Close()
|
||||
|
||||
// Make sure all msgs in outputStream is consumed
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
ts, err := sttbarrier.GetTimeTick()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, Timestamp(10), ts)
|
||||
}
|
||||
|
||||
func TestTt_HardTtBarrierGetTimeTick(t *testing.T) {
|
||||
|
||||
channels := []string{"HardTtBarrierGetTimeTick"}
|
||||
|
||||
ttmsgs := [][2]int{
|
||||
{1, 10},
|
||||
{1, 20},
|
||||
{1, 30},
|
||||
{2, 10},
|
||||
{2, 20},
|
||||
{3, 10},
|
||||
{3, 20},
|
||||
}
|
||||
|
||||
inStream, ttStream := producer(channels, ttmsgs)
|
||||
defer func() {
|
||||
(*inStream).Close()
|
||||
(*ttStream).Close()
|
||||
}()
|
||||
|
||||
peerIds := []UniqueID{1, 2, 3}
|
||||
sttbarrier := NewHardTimeTickBarrier(context.TODO(), ttStream, peerIds)
|
||||
require.NotNil(t, sttbarrier)
|
||||
|
||||
sttbarrier.Start()
|
||||
defer sttbarrier.Close()
|
||||
|
||||
// Make sure all msgs in outputStream is consumed
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
ts, err := sttbarrier.GetTimeTick()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, Timestamp(10), ts)
|
||||
|
||||
ts, err = sttbarrier.GetTimeTick()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, Timestamp(20), ts)
|
||||
|
||||
// ---------------------stuck--------------------------
|
||||
channelsStuck := []string{"HardTtBarrierGetTimeTickStuck"}
|
||||
|
||||
ttmsgsStuck := [][2]int{
|
||||
{1, 10},
|
||||
{2, 10},
|
||||
}
|
||||
|
||||
inStreamStuck, ttStreamStuck := producer(channelsStuck, ttmsgsStuck)
|
||||
defer func() {
|
||||
(*inStreamStuck).Close()
|
||||
(*ttStreamStuck).Close()
|
||||
}()
|
||||
|
||||
peerIdsStuck := []UniqueID{1, 2, 3}
|
||||
sttbarrierStuck := NewHardTimeTickBarrier(context.TODO(), ttStreamStuck, peerIdsStuck)
|
||||
require.NotNil(t, sttbarrierStuck)
|
||||
|
||||
sttbarrierStuck.Start()
|
||||
go func() {
|
||||
time.Sleep(1 * time.Second)
|
||||
sttbarrierStuck.Close()
|
||||
}()
|
||||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// This will stuck
|
||||
ts, err = sttbarrierStuck.GetTimeTick()
|
||||
|
||||
// ---------------------context cancel------------------------
|
||||
channelsCancel := []string{"HardTtBarrierGetTimeTickCancel"}
|
||||
|
||||
ttmsgsCancel := [][2]int{
|
||||
{1, 10},
|
||||
{2, 10},
|
||||
}
|
||||
|
||||
inStreamCancel, ttStreamCancel := producer(channelsCancel, ttmsgsCancel)
|
||||
defer func() {
|
||||
(*inStreamCancel).Close()
|
||||
(*ttStreamCancel).Close()
|
||||
}()
|
||||
|
||||
peerIdsCancel := []UniqueID{1, 2, 3}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
sttbarrierCancel := NewHardTimeTickBarrier(ctx, ttStreamCancel, peerIdsCancel)
|
||||
require.NotNil(t, sttbarrierCancel)
|
||||
|
||||
sttbarrierCancel.Start()
|
||||
go func() {
|
||||
time.Sleep(1 * time.Second)
|
||||
cancel()
|
||||
}()
|
||||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// This will stuck
|
||||
ts, err = sttbarrierCancel.GetTimeTick()
|
||||
|
||||
}
|
|
@ -2,13 +2,10 @@ package timesync
|
|||
|
||||
import "github.com/zilliztech/milvus-distributed/internal/util/typeutil"
|
||||
|
||||
type (
|
||||
UniqueID = typeutil.UniqueID
|
||||
Timestamp = typeutil.Timestamp
|
||||
)
|
||||
type UniqueID = typeutil.UniqueID
|
||||
type Timestamp = typeutil.Timestamp
|
||||
|
||||
type TimeTickBarrier interface {
|
||||
GetTimeTick() (Timestamp, error)
|
||||
GetTimeTick() (Timestamp,error)
|
||||
Start() error
|
||||
Close()
|
||||
}
|
||||
|
|
|
@ -36,19 +36,8 @@ type GlobalTSOAllocator struct {
|
|||
tso *timestampOracle
|
||||
}
|
||||
|
||||
var allocator *GlobalTSOAllocator
|
||||
|
||||
func Init() {
|
||||
InitGlobalTsoAllocator("timestamp", tsoutil.NewTSOKVBase("tso"))
|
||||
}
|
||||
|
||||
func InitGlobalTsoAllocator(key string, base kv.KVBase) {
|
||||
allocator = NewGlobalTSOAllocator(key, base)
|
||||
allocator.Initialize()
|
||||
}
|
||||
|
||||
// NewGlobalTSOAllocator creates a new global TSO allocator.
|
||||
func NewGlobalTSOAllocator(key string, kvBase kv.KVBase) *GlobalTSOAllocator {
|
||||
func NewGlobalTSOAllocator(key string, kvBase kv.KVBase) Allocator {
|
||||
|
||||
var saveInterval time.Duration = 3 * time.Second
|
||||
return &GlobalTSOAllocator{
|
||||
|
@ -63,7 +52,7 @@ func NewGlobalTSOAllocator(key string, kvBase kv.KVBase) *GlobalTSOAllocator {
|
|||
|
||||
// Initialize will initialize the created global TSO allocator.
|
||||
func (gta *GlobalTSOAllocator) Initialize() error {
|
||||
return gta.tso.InitTimestamp()
|
||||
return gta.tso.SyncTimestamp()
|
||||
}
|
||||
|
||||
// UpdateTSO is used to update the TSO in memory and the time window in etcd.
|
||||
|
@ -108,33 +97,7 @@ func (gta *GlobalTSOAllocator) GenerateTSO(count uint32) (uint64, error) {
|
|||
return 0, errors.New("can not get timestamp")
|
||||
}
|
||||
|
||||
func (gta *GlobalTSOAllocator) Alloc(count uint32) (typeutil.Timestamp, error) {
|
||||
//return gta.tso.SyncTimestamp()
|
||||
start, err := gta.GenerateTSO(count)
|
||||
if err != nil {
|
||||
return typeutil.ZeroTimestamp, err
|
||||
}
|
||||
//ret := make([]typeutil.Timestamp, count)
|
||||
//for i:=uint32(0); i < count; i++{
|
||||
// ret[i] = start + uint64(i)
|
||||
//}
|
||||
return start, err
|
||||
}
|
||||
|
||||
func (gta *GlobalTSOAllocator) AllocOne() (typeutil.Timestamp, error) {
|
||||
return gta.GenerateTSO(1)
|
||||
}
|
||||
|
||||
// Reset is used to reset the TSO allocator.
|
||||
func (gta *GlobalTSOAllocator) Reset() {
|
||||
gta.tso.ResetTimestamp()
|
||||
}
|
||||
|
||||
func AllocOne() (typeutil.Timestamp, error) {
|
||||
return allocator.AllocOne()
|
||||
}
|
||||
|
||||
// Reset is used to reset the TSO allocator.
|
||||
func Alloc(count uint32) (typeutil.Timestamp, error) {
|
||||
return allocator.Alloc(count)
|
||||
}
|
||||
|
|
|
@ -1,21 +1,18 @@
|
|||
package tso
|
||||
|
||||
import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv/mockkv"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/tsoutil"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/tsoutil"
|
||||
)
|
||||
|
||||
var GTsoAllocator Allocator
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
conf.LoadConfig("config.yaml")
|
||||
GTsoAllocator = NewGlobalTSOAllocator("timestamp", tsoutil.NewTSOKVBase("tso"))
|
||||
|
||||
GTsoAllocator = NewGlobalTSOAllocator("timestamp", mockkv.NewEtcdKV())
|
||||
exitCode := m.Run()
|
||||
os.Exit(exitCode)
|
||||
}
|
||||
|
@ -31,7 +28,7 @@ func TestGlobalTSOAllocator_GenerateTSO(t *testing.T) {
|
|||
startTs, err := GTsoAllocator.GenerateTSO(perCount)
|
||||
assert.Nil(t, err)
|
||||
lastPhysical, lastLogical := tsoutil.ParseTS(startTs)
|
||||
for i := 0; i < count; i++ {
|
||||
for i:=0;i < count; i++{
|
||||
ts, _ := GTsoAllocator.GenerateTSO(perCount)
|
||||
physical, logical := tsoutil.ParseTS(ts)
|
||||
if lastPhysical == physical {
|
||||
|
@ -44,7 +41,7 @@ func TestGlobalTSOAllocator_GenerateTSO(t *testing.T) {
|
|||
|
||||
func TestGlobalTSOAllocator_SetTSO(t *testing.T) {
|
||||
curTime := time.Now()
|
||||
nextTime := curTime.Add(2 * time.Second)
|
||||
nextTime := curTime.Add(2 * time.Second )
|
||||
physical := nextTime.UnixNano() / int64(time.Millisecond)
|
||||
logical := int64(0)
|
||||
err := GTsoAllocator.SetTSO(tsoutil.ComposeTS(physical, logical))
|
||||
|
|
|
@ -46,8 +46,8 @@ type atomicObject struct {
|
|||
|
||||
// timestampOracle is used to maintain the logic of tso.
|
||||
type timestampOracle struct {
|
||||
key string
|
||||
kvBase kv.KVBase
|
||||
key string
|
||||
kvBase kv.KVBase
|
||||
|
||||
// TODO: remove saveInterval
|
||||
saveInterval time.Duration
|
||||
|
@ -83,27 +83,28 @@ func (t *timestampOracle) saveTimestamp(ts time.Time) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (t *timestampOracle) InitTimestamp() error {
|
||||
// SyncTimestamp is used to synchronize the timestamp.
|
||||
func (t *timestampOracle) SyncTimestamp() error {
|
||||
|
||||
//last, err := t.loadTimestamp()
|
||||
//if err != nil {
|
||||
// return err
|
||||
//}
|
||||
last, err := t.loadTimestamp()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
next := time.Now()
|
||||
|
||||
// If the current system time minus the saved etcd timestamp is less than `updateTimestampGuard`,
|
||||
// the timestamp allocation will start from the saved etcd timestamp temporarily.
|
||||
//if typeutil.SubTimeByWallClock(next, last) < updateTimestampGuard {
|
||||
// next = last.Add(updateTimestampGuard)
|
||||
//}
|
||||
if typeutil.SubTimeByWallClock(next, last) < updateTimestampGuard {
|
||||
next = last.Add(updateTimestampGuard)
|
||||
}
|
||||
|
||||
save := next.Add(t.saveInterval)
|
||||
if err := t.saveTimestamp(save); err != nil {
|
||||
if err = t.saveTimestamp(save); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
//log.Print("sync and save timestamp", zap.Time("last", last), zap.Time("save", save), zap.Time("next", next))
|
||||
log.Print("sync and save timestamp", zap.Time("last", last), zap.Time("save", save), zap.Time("next", next))
|
||||
|
||||
current := &atomicObject{
|
||||
physical: next,
|
||||
|
@ -155,7 +156,7 @@ func (t *timestampOracle) UpdateTimestamp() error {
|
|||
now := time.Now()
|
||||
|
||||
jetLag := typeutil.SubTimeByWallClock(now, prev.physical)
|
||||
if jetLag > 3*UpdateTimestampStep {
|
||||
if jetLag > 3 * UpdateTimestampStep {
|
||||
log.Print("clock offset", zap.Duration("jet-lag", jetLag), zap.Time("prev-physical", prev.physical), zap.Time("now", now))
|
||||
}
|
||||
|
||||
|
@ -196,7 +197,7 @@ func (t *timestampOracle) UpdateTimestamp() error {
|
|||
// ResetTimestamp is used to reset the timestamp.
|
||||
func (t *timestampOracle) ResetTimestamp() {
|
||||
zero := &atomicObject{
|
||||
physical: time.Now(),
|
||||
physical: typeutil.ZeroTime,
|
||||
}
|
||||
atomic.StorePointer(&t.TSO, unsafe.Pointer(zero))
|
||||
}
|
||||
|
|
|
@ -2,12 +2,10 @@ package msgstream
|
|||
|
||||
import (
|
||||
"context"
|
||||
"github.com/zilliztech/milvus-distributed/internal/errors"
|
||||
"log"
|
||||
"sync"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
commonPb "github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
|
||||
"github.com/apache/pulsar-client-go/pulsar"
|
||||
|
@ -24,7 +22,7 @@ type MsgPack struct {
|
|||
Msgs []*TsMsg
|
||||
}
|
||||
|
||||
type RepackFunc func(msgs []*TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, error)
|
||||
type RepackFunc func(msgs []*TsMsg, hashKeys [][]int32) map[int32]*MsgPack
|
||||
|
||||
type MsgStream interface {
|
||||
Start()
|
||||
|
@ -44,18 +42,18 @@ type PulsarMsgStream struct {
|
|||
repackFunc RepackFunc
|
||||
unmarshal *UnmarshalDispatcher
|
||||
receiveBuf chan *MsgPack
|
||||
receiveBufSize int64
|
||||
wait *sync.WaitGroup
|
||||
streamCancel func()
|
||||
}
|
||||
|
||||
func NewPulsarMsgStream(ctx context.Context, receiveBufSize int64) *PulsarMsgStream {
|
||||
streamCtx, streamCancel := context.WithCancel(ctx)
|
||||
stream := &PulsarMsgStream{
|
||||
return &PulsarMsgStream{
|
||||
ctx: streamCtx,
|
||||
streamCancel: streamCancel,
|
||||
receiveBufSize: receiveBufSize,
|
||||
}
|
||||
stream.receiveBuf = make(chan *MsgPack, receiveBufSize)
|
||||
return stream
|
||||
}
|
||||
|
||||
func (ms *PulsarMsgStream) SetPulsarCient(address string) {
|
||||
|
@ -147,23 +145,22 @@ func (ms *PulsarMsgStream) Produce(msgPack *MsgPack) error {
|
|||
}
|
||||
|
||||
var result map[int32]*MsgPack
|
||||
var err error
|
||||
if ms.repackFunc != nil {
|
||||
result, err = ms.repackFunc(tsMsgs, reBucketValues)
|
||||
result = ms.repackFunc(tsMsgs, reBucketValues)
|
||||
} else {
|
||||
msgType := (*tsMsgs[0]).Type()
|
||||
switch msgType {
|
||||
case internalPb.MsgType_kInsert:
|
||||
result, err = insertRepackFunc(tsMsgs, reBucketValues)
|
||||
case internalPb.MsgType_kDelete:
|
||||
result, err = deleteRepackFunc(tsMsgs, reBucketValues)
|
||||
default:
|
||||
result, err = defaultRepackFunc(tsMsgs, reBucketValues)
|
||||
result = make(map[int32]*MsgPack)
|
||||
for i, request := range tsMsgs {
|
||||
keys := reBucketValues[i]
|
||||
for _, channelID := range keys {
|
||||
_, ok := result[channelID]
|
||||
if !ok {
|
||||
msgPack := MsgPack{}
|
||||
result[channelID] = &msgPack
|
||||
}
|
||||
result[channelID].Msgs = append(result[channelID].Msgs, request)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for k, v := range result {
|
||||
for i := 0; i < len(v.Msgs); i++ {
|
||||
mb, err := (*v.Msgs[i]).Marshal(v.Msgs[i])
|
||||
|
@ -218,6 +215,7 @@ func (ms *PulsarMsgStream) Consume() *MsgPack {
|
|||
|
||||
func (ms *PulsarMsgStream) bufMsgPackToChannel() {
|
||||
defer ms.wait.Done()
|
||||
ms.receiveBuf = make(chan *MsgPack, ms.receiveBufSize)
|
||||
for {
|
||||
select {
|
||||
case <-ms.ctx.Done():
|
||||
|
@ -273,8 +271,8 @@ func NewPulsarTtMsgStream(ctx context.Context, receiveBufSize int64) *PulsarTtMs
|
|||
pulsarMsgStream := PulsarMsgStream{
|
||||
ctx: streamCtx,
|
||||
streamCancel: streamCancel,
|
||||
receiveBufSize: receiveBufSize,
|
||||
}
|
||||
pulsarMsgStream.receiveBuf = make(chan *MsgPack, receiveBufSize)
|
||||
return &PulsarTtMsgStream{
|
||||
PulsarMsgStream: pulsarMsgStream,
|
||||
}
|
||||
|
@ -290,6 +288,7 @@ func (ms *PulsarTtMsgStream) Start() {
|
|||
|
||||
func (ms *PulsarTtMsgStream) bufMsgPackToChannel() {
|
||||
defer ms.wait.Done()
|
||||
ms.receiveBuf = make(chan *MsgPack, ms.receiveBufSize)
|
||||
ms.unsolvedBuf = make([]*TsMsg, 0)
|
||||
ms.inputBuf = make([]*TsMsg, 0)
|
||||
for {
|
||||
|
@ -384,113 +383,3 @@ func checkTimeTickMsg(msg map[int]Timestamp) (Timestamp, bool) {
|
|||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func insertRepackFunc(tsMsgs []*TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, error) {
|
||||
result := make(map[int32]*MsgPack)
|
||||
for i, request := range tsMsgs {
|
||||
if (*request).Type() != internalPb.MsgType_kInsert {
|
||||
return nil, errors.New(string("msg's must be Insert"))
|
||||
}
|
||||
insertRequest := (*request).(*InsertMsg)
|
||||
keys := hashKeys[i]
|
||||
|
||||
timestampLen := len(insertRequest.Timestamps)
|
||||
rowIDLen := len(insertRequest.RowIds)
|
||||
rowDataLen := len(insertRequest.RowData)
|
||||
keysLen := len(keys)
|
||||
|
||||
if keysLen != timestampLen || keysLen != rowIDLen || keysLen != rowDataLen {
|
||||
return nil, errors.New(string("the length of hashValue, timestamps, rowIDs, RowData are not equal"))
|
||||
}
|
||||
for index, key := range keys {
|
||||
_, ok := result[key]
|
||||
if !ok {
|
||||
msgPack := MsgPack{}
|
||||
result[key] = &msgPack
|
||||
}
|
||||
|
||||
sliceRequest := internalPb.InsertRequest{
|
||||
MsgType: internalPb.MsgType_kInsert,
|
||||
ReqId: insertRequest.ReqId,
|
||||
CollectionName: insertRequest.CollectionName,
|
||||
PartitionTag: insertRequest.PartitionTag,
|
||||
SegmentId: insertRequest.SegmentId,
|
||||
ChannelId: insertRequest.ChannelId,
|
||||
ProxyId: insertRequest.ProxyId,
|
||||
Timestamps: []uint64{insertRequest.Timestamps[index]},
|
||||
RowIds: []int64{insertRequest.RowIds[index]},
|
||||
RowData: []*commonPb.Blob{insertRequest.RowData[index]},
|
||||
}
|
||||
|
||||
var msg TsMsg = &InsertMsg{
|
||||
InsertRequest: sliceRequest,
|
||||
}
|
||||
|
||||
result[key].Msgs = append(result[key].Msgs, &msg)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func deleteRepackFunc(tsMsgs []*TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, error) {
|
||||
result := make(map[int32]*MsgPack)
|
||||
for i, request := range tsMsgs {
|
||||
if (*request).Type() != internalPb.MsgType_kDelete {
|
||||
return nil, errors.New(string("msg's must be Delete"))
|
||||
}
|
||||
deleteRequest := (*request).(*DeleteMsg)
|
||||
keys := hashKeys[i]
|
||||
|
||||
timestampLen := len(deleteRequest.Timestamps)
|
||||
primaryKeysLen := len(deleteRequest.PrimaryKeys)
|
||||
keysLen := len(keys)
|
||||
|
||||
if keysLen != timestampLen || keysLen != primaryKeysLen {
|
||||
return nil, errors.New(string("the length of hashValue, timestamps, primaryKeys are not equal"))
|
||||
}
|
||||
|
||||
for index, key := range keys {
|
||||
_, ok := result[key]
|
||||
if !ok {
|
||||
msgPack := MsgPack{}
|
||||
result[key] = &msgPack
|
||||
}
|
||||
|
||||
sliceRequest := internalPb.DeleteRequest{
|
||||
MsgType: internalPb.MsgType_kDelete,
|
||||
ReqId: deleteRequest.ReqId,
|
||||
CollectionName: deleteRequest.CollectionName,
|
||||
ChannelId: deleteRequest.ChannelId,
|
||||
ProxyId: deleteRequest.ProxyId,
|
||||
Timestamps: []uint64{deleteRequest.Timestamps[index]},
|
||||
PrimaryKeys: []int64{deleteRequest.PrimaryKeys[index]},
|
||||
}
|
||||
|
||||
var msg TsMsg = &DeleteMsg{
|
||||
DeleteRequest: sliceRequest,
|
||||
}
|
||||
|
||||
result[key].Msgs = append(result[key].Msgs, &msg)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func defaultRepackFunc(tsMsgs []*TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, error) {
|
||||
result := make(map[int32]*MsgPack)
|
||||
for i, request := range tsMsgs {
|
||||
keys := hashKeys[i]
|
||||
if len(keys) != 1 {
|
||||
return nil, errors.New(string("len(msg.hashValue) must equal 1"))
|
||||
}
|
||||
key := keys[0]
|
||||
_, ok := result[key]
|
||||
if !ok {
|
||||
msgPack := MsgPack{}
|
||||
result[key] = &msgPack
|
||||
}
|
||||
|
||||
result[key].Msgs = append(result[key].Msgs, request)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
|
|
@ -3,14 +3,13 @@ package msgstream
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
commonPb "github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
)
|
||||
|
||||
func repackFunc(msgs []*TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, error) {
|
||||
func repackFunc(msgs []*TsMsg, hashKeys [][]int32) map[int32]*MsgPack {
|
||||
result := make(map[int32]*MsgPack)
|
||||
for i, request := range msgs {
|
||||
keys := hashKeys[i]
|
||||
|
@ -23,7 +22,7 @@ func repackFunc(msgs []*TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, error) {
|
|||
result[channelID].Msgs = append(result[channelID].Msgs, request)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
return result
|
||||
}
|
||||
|
||||
func getTsMsg(msgType MsgType, reqID UniqueID, hashValue int32) *TsMsg {
|
||||
|
@ -44,8 +43,6 @@ func getTsMsg(msgType MsgType, reqID UniqueID, hashValue int32) *TsMsg {
|
|||
ChannelId: 1,
|
||||
ProxyId: 1,
|
||||
Timestamps: []Timestamp{1},
|
||||
RowIds: []int64{1},
|
||||
RowData: []*commonPb.Blob{{}},
|
||||
}
|
||||
insertMsg := &InsertMsg{
|
||||
BaseMsg: baseMsg,
|
||||
|
@ -212,11 +209,7 @@ func TestStream_PulsarMsgStream_Insert(t *testing.T) {
|
|||
msgPack.Msgs = append(msgPack.Msgs, getTsMsg(internalPb.MsgType_kInsert, 3, 3))
|
||||
|
||||
inputStream, outputStream := initPulsarStream(pulsarAddress, producerChannels, consumerChannels, consumerSubName)
|
||||
err := (*inputStream).Produce(&msgPack)
|
||||
if err != nil {
|
||||
log.Fatalf("produce error = %v", err)
|
||||
}
|
||||
|
||||
(*inputStream).Produce(&msgPack)
|
||||
receiveMsg(outputStream, len(msgPack.Msgs))
|
||||
(*inputStream).Close()
|
||||
(*outputStream).Close()
|
||||
|
@ -234,10 +227,7 @@ func TestStream_PulsarMsgStream_Delete(t *testing.T) {
|
|||
msgPack.Msgs = append(msgPack.Msgs, getTsMsg(internalPb.MsgType_kDelete, 3, 3))
|
||||
|
||||
inputStream, outputStream := initPulsarStream(pulsarAddress, producerChannels, consumerChannels, consumerSubName)
|
||||
err := (*inputStream).Produce(&msgPack)
|
||||
if err != nil {
|
||||
log.Fatalf("produce error = %v", err)
|
||||
}
|
||||
(*inputStream).Produce(&msgPack)
|
||||
receiveMsg(outputStream, len(msgPack.Msgs))
|
||||
(*inputStream).Close()
|
||||
(*outputStream).Close()
|
||||
|
@ -254,10 +244,7 @@ func TestStream_PulsarMsgStream_Search(t *testing.T) {
|
|||
msgPack.Msgs = append(msgPack.Msgs, getTsMsg(internalPb.MsgType_kSearch, 3, 3))
|
||||
|
||||
inputStream, outputStream := initPulsarStream(pulsarAddress, producerChannels, consumerChannels, consumerSubName)
|
||||
err := (*inputStream).Produce(&msgPack)
|
||||
if err != nil {
|
||||
log.Fatalf("produce error = %v", err)
|
||||
}
|
||||
(*inputStream).Produce(&msgPack)
|
||||
receiveMsg(outputStream, len(msgPack.Msgs))
|
||||
(*inputStream).Close()
|
||||
(*outputStream).Close()
|
||||
|
@ -274,10 +261,7 @@ func TestStream_PulsarMsgStream_SearchResult(t *testing.T) {
|
|||
msgPack.Msgs = append(msgPack.Msgs, getTsMsg(internalPb.MsgType_kSearchResult, 3, 3))
|
||||
|
||||
inputStream, outputStream := initPulsarStream(pulsarAddress, producerChannels, consumerChannels, consumerSubName)
|
||||
err := (*inputStream).Produce(&msgPack)
|
||||
if err != nil {
|
||||
log.Fatalf("produce error = %v", err)
|
||||
}
|
||||
(*inputStream).Produce(&msgPack)
|
||||
receiveMsg(outputStream, len(msgPack.Msgs))
|
||||
(*inputStream).Close()
|
||||
(*outputStream).Close()
|
||||
|
@ -294,10 +278,7 @@ func TestStream_PulsarMsgStream_TimeTick(t *testing.T) {
|
|||
msgPack.Msgs = append(msgPack.Msgs, getTsMsg(internalPb.MsgType_kTimeTick, 3, 3))
|
||||
|
||||
inputStream, outputStream := initPulsarStream(pulsarAddress, producerChannels, consumerChannels, consumerSubName)
|
||||
err := (*inputStream).Produce(&msgPack)
|
||||
if err != nil {
|
||||
log.Fatalf("produce error = %v", err)
|
||||
}
|
||||
(*inputStream).Produce(&msgPack)
|
||||
receiveMsg(outputStream, len(msgPack.Msgs))
|
||||
(*inputStream).Close()
|
||||
(*outputStream).Close()
|
||||
|
@ -314,10 +295,7 @@ func TestStream_PulsarMsgStream_BroadCast(t *testing.T) {
|
|||
msgPack.Msgs = append(msgPack.Msgs, getTsMsg(internalPb.MsgType_kTimeTick, 3, 3))
|
||||
|
||||
inputStream, outputStream := initPulsarStream(pulsarAddress, producerChannels, consumerChannels, consumerSubName)
|
||||
err := (*inputStream).Broadcast(&msgPack)
|
||||
if err != nil {
|
||||
log.Fatalf("produce error = %v", err)
|
||||
}
|
||||
(*inputStream).Broadcast(&msgPack)
|
||||
receiveMsg(outputStream, len(consumerChannels)*len(msgPack.Msgs))
|
||||
(*inputStream).Close()
|
||||
(*outputStream).Close()
|
||||
|
@ -334,164 +312,12 @@ func TestStream_PulsarMsgStream_RepackFunc(t *testing.T) {
|
|||
msgPack.Msgs = append(msgPack.Msgs, getTsMsg(internalPb.MsgType_kInsert, 3, 3))
|
||||
|
||||
inputStream, outputStream := initPulsarStream(pulsarAddress, producerChannels, consumerChannels, consumerSubName, repackFunc)
|
||||
err := (*inputStream).Produce(&msgPack)
|
||||
if err != nil {
|
||||
log.Fatalf("produce error = %v", err)
|
||||
}
|
||||
(*inputStream).Produce(&msgPack)
|
||||
receiveMsg(outputStream, len(msgPack.Msgs))
|
||||
(*inputStream).Close()
|
||||
(*outputStream).Close()
|
||||
}
|
||||
|
||||
func TestStream_PulsarMsgStream_InsertRepackFunc(t *testing.T) {
|
||||
pulsarAddress := "pulsar://localhost:6650"
|
||||
producerChannels := []string{"insert1", "insert2"}
|
||||
consumerChannels := []string{"insert1", "insert2"}
|
||||
consumerSubName := "subInsert"
|
||||
|
||||
baseMsg := BaseMsg{
|
||||
BeginTimestamp: 0,
|
||||
EndTimestamp: 0,
|
||||
HashValues: []int32{1, 3},
|
||||
}
|
||||
|
||||
insertRequest := internalPb.InsertRequest{
|
||||
MsgType: internalPb.MsgType_kInsert,
|
||||
ReqId: 1,
|
||||
CollectionName: "Collection",
|
||||
PartitionTag: "Partition",
|
||||
SegmentId: 1,
|
||||
ChannelId: 1,
|
||||
ProxyId: 1,
|
||||
Timestamps: []Timestamp{1, 1},
|
||||
RowIds: []int64{1, 3},
|
||||
RowData: []*commonPb.Blob{{}, {}},
|
||||
}
|
||||
insertMsg := &InsertMsg{
|
||||
BaseMsg: baseMsg,
|
||||
InsertRequest: insertRequest,
|
||||
}
|
||||
var tsMsg TsMsg = insertMsg
|
||||
msgPack := MsgPack{}
|
||||
msgPack.Msgs = append(msgPack.Msgs, &tsMsg)
|
||||
|
||||
inputStream := NewPulsarMsgStream(context.Background(), 100)
|
||||
inputStream.SetPulsarCient(pulsarAddress)
|
||||
inputStream.CreatePulsarProducers(producerChannels)
|
||||
inputStream.Start()
|
||||
|
||||
outputStream := NewPulsarMsgStream(context.Background(), 100)
|
||||
outputStream.SetPulsarCient(pulsarAddress)
|
||||
unmarshalDispatcher := NewUnmarshalDispatcher()
|
||||
outputStream.CreatePulsarConsumers(consumerChannels, consumerSubName, unmarshalDispatcher, 100)
|
||||
outputStream.Start()
|
||||
var output MsgStream = outputStream
|
||||
|
||||
err := (*inputStream).Produce(&msgPack)
|
||||
if err != nil {
|
||||
log.Fatalf("produce error = %v", err)
|
||||
}
|
||||
receiveMsg(&output, len(msgPack.Msgs)*2)
|
||||
(*inputStream).Close()
|
||||
(*outputStream).Close()
|
||||
}
|
||||
|
||||
func TestStream_PulsarMsgStream_DeleteRepackFunc(t *testing.T) {
|
||||
pulsarAddress := "pulsar://localhost:6650"
|
||||
producerChannels := []string{"insert1", "insert2"}
|
||||
consumerChannels := []string{"insert1", "insert2"}
|
||||
consumerSubName := "subInsert"
|
||||
|
||||
baseMsg := BaseMsg{
|
||||
BeginTimestamp: 0,
|
||||
EndTimestamp: 0,
|
||||
HashValues: []int32{1, 3},
|
||||
}
|
||||
|
||||
deleteRequest := internalPb.DeleteRequest{
|
||||
MsgType: internalPb.MsgType_kDelete,
|
||||
ReqId: 1,
|
||||
CollectionName: "Collection",
|
||||
ChannelId: 1,
|
||||
ProxyId: 1,
|
||||
Timestamps: []Timestamp{1, 1},
|
||||
PrimaryKeys: []int64{1, 3},
|
||||
}
|
||||
deleteMsg := &DeleteMsg{
|
||||
BaseMsg: baseMsg,
|
||||
DeleteRequest: deleteRequest,
|
||||
}
|
||||
var tsMsg TsMsg = deleteMsg
|
||||
msgPack := MsgPack{}
|
||||
msgPack.Msgs = append(msgPack.Msgs, &tsMsg)
|
||||
|
||||
inputStream := NewPulsarMsgStream(context.Background(), 100)
|
||||
inputStream.SetPulsarCient(pulsarAddress)
|
||||
inputStream.CreatePulsarProducers(producerChannels)
|
||||
inputStream.Start()
|
||||
|
||||
outputStream := NewPulsarMsgStream(context.Background(), 100)
|
||||
outputStream.SetPulsarCient(pulsarAddress)
|
||||
unmarshalDispatcher := NewUnmarshalDispatcher()
|
||||
outputStream.CreatePulsarConsumers(consumerChannels, consumerSubName, unmarshalDispatcher, 100)
|
||||
outputStream.Start()
|
||||
var output MsgStream = outputStream
|
||||
|
||||
err := (*inputStream).Produce(&msgPack)
|
||||
if err != nil {
|
||||
log.Fatalf("produce error = %v", err)
|
||||
}
|
||||
receiveMsg(&output, len(msgPack.Msgs)*2)
|
||||
(*inputStream).Close()
|
||||
(*outputStream).Close()
|
||||
}
|
||||
|
||||
func TestStream_PulsarMsgStream_DefaultRepackFunc(t *testing.T) {
|
||||
pulsarAddress := "pulsar://localhost:6650"
|
||||
producerChannels := []string{"insert1", "insert2"}
|
||||
consumerChannels := []string{"insert1", "insert2"}
|
||||
consumerSubName := "subInsert"
|
||||
|
||||
baseMsg := BaseMsg{
|
||||
BeginTimestamp: 0,
|
||||
EndTimestamp: 0,
|
||||
HashValues: []int32{1},
|
||||
}
|
||||
|
||||
timeTickRequest := internalPb.TimeTickMsg{
|
||||
MsgType: internalPb.MsgType_kTimeTick,
|
||||
PeerId: int64(1),
|
||||
Timestamp: uint64(1),
|
||||
}
|
||||
timeTick := &TimeTickMsg{
|
||||
BaseMsg: baseMsg,
|
||||
TimeTickMsg: timeTickRequest,
|
||||
}
|
||||
var tsMsg TsMsg = timeTick
|
||||
msgPack := MsgPack{}
|
||||
msgPack.Msgs = append(msgPack.Msgs, &tsMsg)
|
||||
|
||||
inputStream := NewPulsarMsgStream(context.Background(), 100)
|
||||
inputStream.SetPulsarCient(pulsarAddress)
|
||||
inputStream.CreatePulsarProducers(producerChannels)
|
||||
inputStream.Start()
|
||||
|
||||
outputStream := NewPulsarMsgStream(context.Background(), 100)
|
||||
outputStream.SetPulsarCient(pulsarAddress)
|
||||
unmarshalDispatcher := NewUnmarshalDispatcher()
|
||||
outputStream.CreatePulsarConsumers(consumerChannels, consumerSubName, unmarshalDispatcher, 100)
|
||||
outputStream.Start()
|
||||
var output MsgStream = outputStream
|
||||
|
||||
err := (*inputStream).Produce(&msgPack)
|
||||
if err != nil {
|
||||
log.Fatalf("produce error = %v", err)
|
||||
}
|
||||
receiveMsg(&output, len(msgPack.Msgs))
|
||||
(*inputStream).Close()
|
||||
(*outputStream).Close()
|
||||
}
|
||||
|
||||
func TestStream_PulsarTtMsgStream_Insert(t *testing.T) {
|
||||
pulsarAddress := "pulsar://localhost:6650"
|
||||
producerChannels := []string{"insert1", "insert2"}
|
||||
|
@ -509,18 +335,9 @@ func TestStream_PulsarTtMsgStream_Insert(t *testing.T) {
|
|||
msgPack2.Msgs = append(msgPack2.Msgs, getTimeTickMsg(5, 5, 5))
|
||||
|
||||
inputStream, outputStream := initPulsarTtStream(pulsarAddress, producerChannels, consumerChannels, consumerSubName)
|
||||
err := (*inputStream).Broadcast(&msgPack0)
|
||||
if err != nil {
|
||||
log.Fatalf("broadcast error = %v", err)
|
||||
}
|
||||
err = (*inputStream).Produce(&msgPack1)
|
||||
if err != nil {
|
||||
log.Fatalf("produce error = %v", err)
|
||||
}
|
||||
err = (*inputStream).Broadcast(&msgPack2)
|
||||
if err != nil {
|
||||
log.Fatalf("broadcast error = %v", err)
|
||||
}
|
||||
(*inputStream).Broadcast(&msgPack0)
|
||||
(*inputStream).Produce(&msgPack1)
|
||||
(*inputStream).Broadcast(&msgPack2)
|
||||
receiveMsg(outputStream, len(msgPack1.Msgs))
|
||||
outputTtStream := (*outputStream).(*PulsarTtMsgStream)
|
||||
fmt.Printf("timestamp = %v", outputTtStream.lastTimeStamp)
|
||||
|
|
|
@ -2,10 +2,7 @@ package msgstream
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
commonPb "github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
|
@ -40,53 +37,6 @@ func (tt *InsertTask) Unmarshal(input []byte) (*TsMsg, error) {
|
|||
return &tsMsg, nil
|
||||
}
|
||||
|
||||
func newRepackFunc(tsMsgs []*TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, error) {
|
||||
result := make(map[int32]*MsgPack)
|
||||
for i, request := range tsMsgs {
|
||||
if (*request).Type() != internalPb.MsgType_kInsert {
|
||||
return nil, errors.New(string("msg's must be Insert"))
|
||||
}
|
||||
insertRequest := (*request).(*InsertTask).InsertRequest
|
||||
keys := hashKeys[i]
|
||||
|
||||
timestampLen := len(insertRequest.Timestamps)
|
||||
rowIDLen := len(insertRequest.RowIds)
|
||||
rowDataLen := len(insertRequest.RowData)
|
||||
keysLen := len(keys)
|
||||
|
||||
if keysLen != timestampLen || keysLen != rowIDLen || keysLen != rowDataLen {
|
||||
return nil, errors.New(string("the length of hashValue, timestamps, rowIDs, RowData are not equal"))
|
||||
}
|
||||
for index, key := range keys {
|
||||
_, ok := result[key]
|
||||
if !ok {
|
||||
msgPack := MsgPack{}
|
||||
result[key] = &msgPack
|
||||
}
|
||||
|
||||
sliceRequest := internalPb.InsertRequest{
|
||||
MsgType: internalPb.MsgType_kInsert,
|
||||
ReqId: insertRequest.ReqId,
|
||||
CollectionName: insertRequest.CollectionName,
|
||||
PartitionTag: insertRequest.PartitionTag,
|
||||
SegmentId: insertRequest.SegmentId,
|
||||
ChannelId: insertRequest.ChannelId,
|
||||
ProxyId: insertRequest.ProxyId,
|
||||
Timestamps: []uint64{insertRequest.Timestamps[index]},
|
||||
RowIds: []int64{insertRequest.RowIds[index]},
|
||||
RowData: []*commonPb.Blob{insertRequest.RowData[index]},
|
||||
}
|
||||
|
||||
var msg TsMsg = &InsertTask{
|
||||
InsertMsg: InsertMsg{InsertRequest: sliceRequest},
|
||||
}
|
||||
|
||||
result[key].Msgs = append(result[key].Msgs, &msg)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func getMsg(reqID UniqueID, hashValue int32) *TsMsg {
|
||||
var tsMsg TsMsg
|
||||
baseMsg := BaseMsg{
|
||||
|
@ -103,8 +53,6 @@ func getMsg(reqID UniqueID, hashValue int32) *TsMsg {
|
|||
ChannelId: 1,
|
||||
ProxyId: 1,
|
||||
Timestamps: []Timestamp{1},
|
||||
RowIds: []int64{1},
|
||||
RowData: []*commonPb.Blob{{}},
|
||||
}
|
||||
insertMsg := InsertMsg{
|
||||
BaseMsg: baseMsg,
|
||||
|
@ -131,7 +79,6 @@ func TestStream_task_Insert(t *testing.T) {
|
|||
inputStream := NewPulsarMsgStream(context.Background(), 100)
|
||||
inputStream.SetPulsarCient(pulsarAddress)
|
||||
inputStream.CreatePulsarProducers(producerChannels)
|
||||
inputStream.SetRepackFunc(newRepackFunc)
|
||||
inputStream.Start()
|
||||
|
||||
outputStream := NewPulsarMsgStream(context.Background(), 100)
|
||||
|
@ -142,10 +89,7 @@ func TestStream_task_Insert(t *testing.T) {
|
|||
outputStream.CreatePulsarConsumers(consumerChannels, consumerSubName, unmarshalDispatcher, 100)
|
||||
outputStream.Start()
|
||||
|
||||
err := inputStream.Produce(&msgPack)
|
||||
if err != nil {
|
||||
log.Fatalf("produce error = %v", err)
|
||||
}
|
||||
inputStream.Produce(&msgPack)
|
||||
receiveCount := 0
|
||||
for {
|
||||
result := (*outputStream).Consume()
|
||||
|
|
|
@ -3,7 +3,6 @@ package msgstream
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
|
@ -48,10 +47,7 @@ func TestStream_unmarshal_Insert(t *testing.T) {
|
|||
outputStream.CreatePulsarConsumers(consumerChannels, consumerSubName, unmarshalDispatcher, 100)
|
||||
outputStream.Start()
|
||||
|
||||
err := inputStream.Produce(&msgPack)
|
||||
if err != nil {
|
||||
log.Fatalf("produce error = %v", err)
|
||||
}
|
||||
inputStream.Produce(&msgPack)
|
||||
receiveCount := 0
|
||||
for {
|
||||
result := (*outputStream).Consume()
|
||||
|
|
|
@ -3,7 +3,7 @@ package proxy
|
|||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/gogo/protobuf/proto"
|
||||
"github.com/zilliztech/milvus-distributed/internal/msgstream"
|
||||
"log"
|
||||
|
||||
|
@ -35,18 +35,21 @@ func (p *Proxy) Insert(ctx context.Context, in *servicepb.RowBatch) (*servicepb.
|
|||
|
||||
defer it.cancel()
|
||||
|
||||
p.taskSch.DmQueue.Enqueue(it)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("insert timeout!")
|
||||
return &servicepb.IntegerRangeResponse{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "insert timeout!",
|
||||
},
|
||||
}, errors.New("insert timeout!")
|
||||
case result := <-it.resultChan:
|
||||
return result, nil
|
||||
var t task = it
|
||||
p.taskSch.DmQueue.Enqueue(&t)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("insert timeout!")
|
||||
return &servicepb.IntegerRangeResponse{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "insert timeout!",
|
||||
},
|
||||
}, errors.New("insert timeout!")
|
||||
case result := <-it.resultChan:
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -66,16 +69,19 @@ func (p *Proxy) CreateCollection(ctx context.Context, req *schemapb.CollectionSc
|
|||
cct.ctx, cct.cancel = context.WithCancel(ctx)
|
||||
defer cct.cancel()
|
||||
|
||||
p.taskSch.DdQueue.Enqueue(cct)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("create collection timeout!")
|
||||
return &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "create collection timeout!",
|
||||
}, errors.New("create collection timeout!")
|
||||
case result := <-cct.resultChan:
|
||||
return result, nil
|
||||
var t task = cct
|
||||
p.taskSch.DdQueue.Enqueue(&t)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("create collection timeout!")
|
||||
return &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "create collection timeout!",
|
||||
}, errors.New("create collection timeout!")
|
||||
case result := <-cct.resultChan:
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -96,18 +102,21 @@ func (p *Proxy) Search(ctx context.Context, req *servicepb.Query) (*servicepb.Qu
|
|||
qt.SearchRequest.Query.Value = queryBytes
|
||||
defer qt.cancel()
|
||||
|
||||
p.taskSch.DqQueue.Enqueue(qt)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("query timeout!")
|
||||
return &servicepb.QueryResult{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "query timeout!",
|
||||
},
|
||||
}, errors.New("query timeout!")
|
||||
case result := <-qt.resultChan:
|
||||
return result, nil
|
||||
var t task = qt
|
||||
p.taskSch.DqQueue.Enqueue(&t)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("query timeout!")
|
||||
return &servicepb.QueryResult{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "query timeout!",
|
||||
},
|
||||
}, errors.New("query timeout!")
|
||||
case result := <-qt.resultChan:
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -125,16 +134,19 @@ func (p *Proxy) DropCollection(ctx context.Context, req *servicepb.CollectionNam
|
|||
dct.ctx, dct.cancel = context.WithCancel(ctx)
|
||||
defer dct.cancel()
|
||||
|
||||
p.taskSch.DdQueue.Enqueue(dct)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("create collection timeout!")
|
||||
return &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "create collection timeout!",
|
||||
}, errors.New("create collection timeout!")
|
||||
case result := <-dct.resultChan:
|
||||
return result, nil
|
||||
var t task = dct
|
||||
p.taskSch.DdQueue.Enqueue(&t)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("create collection timeout!")
|
||||
return &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "create collection timeout!",
|
||||
}, errors.New("create collection timeout!")
|
||||
case result := <-dct.resultChan:
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -152,19 +164,22 @@ func (p *Proxy) HasCollection(ctx context.Context, req *servicepb.CollectionName
|
|||
hct.ctx, hct.cancel = context.WithCancel(ctx)
|
||||
defer hct.cancel()
|
||||
|
||||
p.taskSch.DqQueue.Enqueue(hct)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("has collection timeout!")
|
||||
return &servicepb.BoolResponse{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "has collection timeout!",
|
||||
},
|
||||
Value: false,
|
||||
}, errors.New("has collection timeout!")
|
||||
case result := <-hct.resultChan:
|
||||
return result, nil
|
||||
var t task = hct
|
||||
p.taskSch.DqQueue.Enqueue(&t)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("has collection timeout!")
|
||||
return &servicepb.BoolResponse{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "has collection timeout!",
|
||||
},
|
||||
Value: false,
|
||||
}, errors.New("has collection timeout!")
|
||||
case result := <-hct.resultChan:
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -182,18 +197,21 @@ func (p *Proxy) DescribeCollection(ctx context.Context, req *servicepb.Collectio
|
|||
dct.ctx, dct.cancel = context.WithCancel(ctx)
|
||||
defer dct.cancel()
|
||||
|
||||
p.taskSch.DqQueue.Enqueue(dct)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("has collection timeout!")
|
||||
return &servicepb.CollectionDescription{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "describe collection timeout!",
|
||||
},
|
||||
}, errors.New("describe collection timeout!")
|
||||
case result := <-dct.resultChan:
|
||||
return result, nil
|
||||
var t task = dct
|
||||
p.taskSch.DqQueue.Enqueue(&t)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("has collection timeout!")
|
||||
return &servicepb.CollectionDescription{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "describe collection timeout!",
|
||||
},
|
||||
}, errors.New("describe collection timeout!")
|
||||
case result := <-dct.resultChan:
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -210,18 +228,21 @@ func (p *Proxy) ShowCollections(ctx context.Context, req *commonpb.Empty) (*serv
|
|||
sct.ctx, sct.cancel = context.WithCancel(ctx)
|
||||
defer sct.cancel()
|
||||
|
||||
p.taskSch.DqQueue.Enqueue(sct)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("show collections timeout!")
|
||||
return &servicepb.StringListResponse{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "show collections timeout!",
|
||||
},
|
||||
}, errors.New("show collections timeout!")
|
||||
case result := <-sct.resultChan:
|
||||
return result, nil
|
||||
var t task = sct
|
||||
p.taskSch.DqQueue.Enqueue(&t)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("show collections timeout!")
|
||||
return &servicepb.StringListResponse{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "show collections timeout!",
|
||||
},
|
||||
}, errors.New("show collections timeout!")
|
||||
case result := <-sct.resultChan:
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -2,7 +2,6 @@ package proxy
|
|||
|
||||
import (
|
||||
"context"
|
||||
"google.golang.org/grpc"
|
||||
"log"
|
||||
"math/rand"
|
||||
"net"
|
||||
|
@ -15,6 +14,7 @@ import (
|
|||
"github.com/zilliztech/milvus-distributed/internal/proto/masterpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/servicepb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
type UniqueID = typeutil.UniqueID
|
||||
|
@ -157,7 +157,7 @@ func (p *Proxy) queryResultLoop() {
|
|||
if len(queryResultBuf[reqId]) == 4 {
|
||||
// TODO: use the number of query node instead
|
||||
t := p.taskSch.getTaskByReqId(reqId)
|
||||
qt := t.(*QueryTask)
|
||||
qt := (*t).(*QueryTask)
|
||||
qt.resultBuf <- queryResultBuf[reqId]
|
||||
delete(queryResultBuf, reqId)
|
||||
}
|
||||
|
|
|
@ -11,7 +11,7 @@ import (
|
|||
|
||||
type BaseTaskQueue struct {
|
||||
unissuedTasks *list.List
|
||||
activeTasks map[Timestamp]task
|
||||
activeTasks map[Timestamp]*task
|
||||
utLock sync.Mutex
|
||||
atLock sync.Mutex
|
||||
}
|
||||
|
@ -24,23 +24,23 @@ func (queue *BaseTaskQueue) Empty() bool {
|
|||
return queue.unissuedTasks.Len() <= 0 && len(queue.activeTasks) <= 0
|
||||
}
|
||||
|
||||
func (queue *BaseTaskQueue) AddUnissuedTask(t task) {
|
||||
func (queue *BaseTaskQueue) AddUnissuedTask(t *task) {
|
||||
queue.utLock.Lock()
|
||||
defer queue.utLock.Unlock()
|
||||
queue.unissuedTasks.PushBack(t)
|
||||
}
|
||||
|
||||
func (queue *BaseTaskQueue) FrontUnissuedTask() task {
|
||||
func (queue *BaseTaskQueue) FrontUnissuedTask() *task {
|
||||
queue.utLock.Lock()
|
||||
defer queue.utLock.Unlock()
|
||||
if queue.unissuedTasks.Len() <= 0 {
|
||||
log.Fatal("sorry, but the unissued task list is empty!")
|
||||
return nil
|
||||
}
|
||||
return queue.unissuedTasks.Front().Value.(task)
|
||||
return queue.unissuedTasks.Front().Value.(*task)
|
||||
}
|
||||
|
||||
func (queue *BaseTaskQueue) PopUnissuedTask() task {
|
||||
func (queue *BaseTaskQueue) PopUnissuedTask() *task {
|
||||
queue.utLock.Lock()
|
||||
defer queue.utLock.Unlock()
|
||||
if queue.unissuedTasks.Len() <= 0 {
|
||||
|
@ -48,13 +48,13 @@ func (queue *BaseTaskQueue) PopUnissuedTask() task {
|
|||
return nil
|
||||
}
|
||||
ft := queue.unissuedTasks.Front()
|
||||
return queue.unissuedTasks.Remove(ft).(task)
|
||||
return queue.unissuedTasks.Remove(ft).(*task)
|
||||
}
|
||||
|
||||
func (queue *BaseTaskQueue) AddActiveTask(t task) {
|
||||
func (queue *BaseTaskQueue) AddActiveTask(t *task) {
|
||||
queue.atLock.Lock()
|
||||
defer queue.atLock.Lock()
|
||||
ts := t.EndTs()
|
||||
ts := (*t).EndTs()
|
||||
_, ok := queue.activeTasks[ts]
|
||||
if ok {
|
||||
log.Fatalf("task with timestamp %v already in active task list!", ts)
|
||||
|
@ -62,7 +62,7 @@ func (queue *BaseTaskQueue) AddActiveTask(t task) {
|
|||
queue.activeTasks[ts] = t
|
||||
}
|
||||
|
||||
func (queue *BaseTaskQueue) PopActiveTask(ts Timestamp) task {
|
||||
func (queue *BaseTaskQueue) PopActiveTask(ts Timestamp) *task {
|
||||
queue.atLock.Lock()
|
||||
defer queue.atLock.Lock()
|
||||
t, ok := queue.activeTasks[ts]
|
||||
|
@ -74,19 +74,19 @@ func (queue *BaseTaskQueue) PopActiveTask(ts Timestamp) task {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (queue *BaseTaskQueue) getTaskByReqId(reqId UniqueID) task {
|
||||
func (queue *BaseTaskQueue) getTaskByReqId(reqId UniqueID) *task {
|
||||
queue.utLock.Lock()
|
||||
defer queue.utLock.Lock()
|
||||
for e := queue.unissuedTasks.Front(); e != nil; e = e.Next() {
|
||||
if e.Value.(task).Id() == reqId {
|
||||
return e.Value.(task)
|
||||
if (*(e.Value.(*task))).Id() == reqId {
|
||||
return e.Value.(*task)
|
||||
}
|
||||
}
|
||||
|
||||
queue.atLock.Lock()
|
||||
defer queue.atLock.Unlock()
|
||||
for ats := range queue.activeTasks {
|
||||
if queue.activeTasks[ats].Id() == reqId {
|
||||
if (*(queue.activeTasks[ats])).Id() == reqId {
|
||||
return queue.activeTasks[ats]
|
||||
}
|
||||
}
|
||||
|
@ -98,7 +98,7 @@ func (queue *BaseTaskQueue) TaskDoneTest(ts Timestamp) bool {
|
|||
queue.utLock.Lock()
|
||||
defer queue.utLock.Unlock()
|
||||
for e := queue.unissuedTasks.Front(); e != nil; e = e.Next() {
|
||||
if e.Value.(task).EndTs() >= ts {
|
||||
if (*(e.Value.(*task))).EndTs() >= ts {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
@ -114,20 +114,20 @@ func (queue *BaseTaskQueue) TaskDoneTest(ts Timestamp) bool {
|
|||
return true
|
||||
}
|
||||
|
||||
type DdTaskQueue struct {
|
||||
type ddTaskQueue struct {
|
||||
BaseTaskQueue
|
||||
lock sync.Mutex
|
||||
}
|
||||
|
||||
type DmTaskQueue struct {
|
||||
type dmTaskQueue struct {
|
||||
BaseTaskQueue
|
||||
}
|
||||
|
||||
type DqTaskQueue struct {
|
||||
type dqTaskQueue struct {
|
||||
BaseTaskQueue
|
||||
}
|
||||
|
||||
func (queue *DdTaskQueue) Enqueue(t task) error {
|
||||
func (queue *ddTaskQueue) Enqueue(t *task) error {
|
||||
queue.lock.Lock()
|
||||
defer queue.lock.Unlock()
|
||||
// TODO: set Ts, ReqId, ProxyId
|
||||
|
@ -135,49 +135,22 @@ func (queue *DdTaskQueue) Enqueue(t task) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (queue *DmTaskQueue) Enqueue(t task) error {
|
||||
func (queue *dmTaskQueue) Enqueue(t *task) error {
|
||||
// TODO: set Ts, ReqId, ProxyId
|
||||
queue.AddUnissuedTask(t)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (queue *DqTaskQueue) Enqueue(t task) error {
|
||||
func (queue *dqTaskQueue) Enqueue(t *task) error {
|
||||
// TODO: set Ts, ReqId, ProxyId
|
||||
queue.AddUnissuedTask(t)
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewDdTaskQueue() *DdTaskQueue {
|
||||
return &DdTaskQueue{
|
||||
BaseTaskQueue: BaseTaskQueue{
|
||||
unissuedTasks: list.New(),
|
||||
activeTasks: make(map[Timestamp]task),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func NewDmTaskQueue() *DmTaskQueue {
|
||||
return &DmTaskQueue{
|
||||
BaseTaskQueue: BaseTaskQueue{
|
||||
unissuedTasks: list.New(),
|
||||
activeTasks: make(map[Timestamp]task),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func NewDqTaskQueue() *DqTaskQueue {
|
||||
return &DqTaskQueue{
|
||||
BaseTaskQueue: BaseTaskQueue{
|
||||
unissuedTasks: list.New(),
|
||||
activeTasks: make(map[Timestamp]task),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type TaskScheduler struct {
|
||||
DdQueue *DdTaskQueue
|
||||
DmQueue *DmTaskQueue
|
||||
DqQueue *DqTaskQueue
|
||||
DdQueue *ddTaskQueue
|
||||
DmQueue *dmTaskQueue
|
||||
DqQueue *dqTaskQueue
|
||||
|
||||
idAllocator *allocator.IdAllocator
|
||||
tsoAllocator *allocator.TimestampAllocator
|
||||
|
@ -192,9 +165,6 @@ func NewTaskScheduler(ctx context.Context,
|
|||
tsoAllocator *allocator.TimestampAllocator) (*TaskScheduler, error) {
|
||||
ctx1, cancel := context.WithCancel(ctx)
|
||||
s := &TaskScheduler{
|
||||
DdQueue: NewDdTaskQueue(),
|
||||
DmQueue: NewDmTaskQueue(),
|
||||
DqQueue: NewDqTaskQueue(),
|
||||
idAllocator: idAllocator,
|
||||
tsoAllocator: tsoAllocator,
|
||||
ctx: ctx1,
|
||||
|
@ -204,19 +174,19 @@ func NewTaskScheduler(ctx context.Context,
|
|||
return s, nil
|
||||
}
|
||||
|
||||
func (sched *TaskScheduler) scheduleDdTask() task {
|
||||
func (sched *TaskScheduler) scheduleDdTask() *task {
|
||||
return sched.DdQueue.PopUnissuedTask()
|
||||
}
|
||||
|
||||
func (sched *TaskScheduler) scheduleDmTask() task {
|
||||
func (sched *TaskScheduler) scheduleDmTask() *task {
|
||||
return sched.DmQueue.PopUnissuedTask()
|
||||
}
|
||||
|
||||
func (sched *TaskScheduler) scheduleDqTask() task {
|
||||
func (sched *TaskScheduler) scheduleDqTask() *task {
|
||||
return sched.DqQueue.PopUnissuedTask()
|
||||
}
|
||||
|
||||
func (sched *TaskScheduler) getTaskByReqId(reqId UniqueID) task {
|
||||
func (sched *TaskScheduler) getTaskByReqId(reqId UniqueID) *task {
|
||||
if t := sched.DdQueue.getTaskByReqId(reqId); t != nil {
|
||||
return t
|
||||
}
|
||||
|
@ -241,22 +211,22 @@ func (sched *TaskScheduler) definitionLoop() {
|
|||
//sched.DdQueue.atLock.Lock()
|
||||
t := sched.scheduleDdTask()
|
||||
|
||||
err := t.PreExecute()
|
||||
err := (*t).PreExecute()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = t.Execute()
|
||||
err = (*t).Execute()
|
||||
if err != nil {
|
||||
log.Printf("execute definition task failed, error = %v", err)
|
||||
}
|
||||
t.Notify(err)
|
||||
(*t).Notify(err)
|
||||
|
||||
sched.DdQueue.AddActiveTask(t)
|
||||
|
||||
t.WaitToFinish()
|
||||
t.PostExecute()
|
||||
(*t).WaitToFinish()
|
||||
(*t).PostExecute()
|
||||
|
||||
sched.DdQueue.PopActiveTask(t.EndTs())
|
||||
sched.DdQueue.PopActiveTask((*t).EndTs())
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -272,27 +242,27 @@ func (sched *TaskScheduler) manipulationLoop() {
|
|||
sched.DmQueue.atLock.Lock()
|
||||
t := sched.scheduleDmTask()
|
||||
|
||||
if err := t.PreExecute(); err != nil {
|
||||
if err := (*t).PreExecute(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
err := t.Execute()
|
||||
err := (*t).Execute()
|
||||
if err != nil {
|
||||
log.Printf("execute manipulation task failed, error = %v", err)
|
||||
}
|
||||
t.Notify(err)
|
||||
(*t).Notify(err)
|
||||
}()
|
||||
|
||||
sched.DmQueue.AddActiveTask(t)
|
||||
sched.DmQueue.atLock.Unlock()
|
||||
|
||||
go func() {
|
||||
t.WaitToFinish()
|
||||
t.PostExecute()
|
||||
(*t).WaitToFinish()
|
||||
(*t).PostExecute()
|
||||
|
||||
// remove from active list
|
||||
sched.DmQueue.PopActiveTask(t.EndTs())
|
||||
sched.DmQueue.PopActiveTask((*t).EndTs())
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
@ -309,27 +279,27 @@ func (sched *TaskScheduler) queryLoop() {
|
|||
sched.DqQueue.atLock.Lock()
|
||||
t := sched.scheduleDqTask()
|
||||
|
||||
if err := t.PreExecute(); err != nil {
|
||||
if err := (*t).PreExecute(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
err := t.Execute()
|
||||
err := (*t).Execute()
|
||||
if err != nil {
|
||||
log.Printf("execute query task failed, error = %v", err)
|
||||
}
|
||||
t.Notify(err)
|
||||
(*t).Notify(err)
|
||||
}()
|
||||
|
||||
sched.DqQueue.AddActiveTask(t)
|
||||
sched.DqQueue.atLock.Unlock()
|
||||
|
||||
go func() {
|
||||
t.WaitToFinish()
|
||||
t.PostExecute()
|
||||
(*t).WaitToFinish()
|
||||
(*t).PostExecute()
|
||||
|
||||
// remove from active list
|
||||
sched.DqQueue.PopActiveTask(t.EndTs())
|
||||
sched.DqQueue.PopActiveTask((*t).EndTs())
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
|
|
@ -51,6 +51,7 @@ func newTimeTick(ctx context.Context, tsoAllocator *allocator.TimestampAllocator
|
|||
return t
|
||||
}
|
||||
|
||||
|
||||
func (tt *timeTick) tick() error {
|
||||
|
||||
if tt.lastTick == tt.currentTick {
|
||||
|
|
|
@ -33,7 +33,7 @@ func TestTimeTick(t *testing.T) {
|
|||
tt := timeTick{
|
||||
interval: 200,
|
||||
pulsarProducer: producer,
|
||||
peerID: 1,
|
||||
peerID: 1,
|
||||
ctx: ctx,
|
||||
areRequestsDelivered: func(ts Timestamp) bool { return true },
|
||||
}
|
||||
|
|
|
@ -77,9 +77,9 @@ func (container *ColSegContainer) getCollectionByName(collectionName string) (*C
|
|||
}
|
||||
|
||||
//----------------------------------------------------------------------------------------------------- partition
|
||||
func (container *ColSegContainer) addPartition(collection *Collection, partitionTag string) (*Partition, error) {
|
||||
func (container *ColSegContainer) addPartition(collection *Collection, partitionTag string) error {
|
||||
if collection == nil {
|
||||
return nil, errors.New("null collection")
|
||||
return errors.New("null collection")
|
||||
}
|
||||
|
||||
var newPartition = newPartition(partitionTag)
|
||||
|
@ -87,11 +87,11 @@ func (container *ColSegContainer) addPartition(collection *Collection, partition
|
|||
for _, col := range container.collections {
|
||||
if col.Name() == collection.Name() {
|
||||
*col.Partitions() = append(*col.Partitions(), newPartition)
|
||||
return newPartition, nil
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.New("cannot find collection, name = " + collection.Name())
|
||||
return errors.New("cannot find collection, name = " + collection.Name())
|
||||
}
|
||||
|
||||
func (container *ColSegContainer) removePartition(partition *Partition) error {
|
||||
|
@ -138,13 +138,13 @@ func (container *ColSegContainer) getPartitionByTag(partitionTag string) (*Parti
|
|||
}
|
||||
|
||||
//----------------------------------------------------------------------------------------------------- segment
|
||||
func (container *ColSegContainer) addSegment(collection *Collection, partition *Partition, segmentID int64) (*Segment, error) {
|
||||
func (container *ColSegContainer) addSegment(collection *Collection, partition *Partition, segmentID int64) error {
|
||||
if collection == nil {
|
||||
return nil, errors.New("null collection")
|
||||
return errors.New("null collection")
|
||||
}
|
||||
|
||||
if partition == nil {
|
||||
return nil, errors.New("null partition")
|
||||
return errors.New("null partition")
|
||||
}
|
||||
|
||||
var newSegment = newSegment(collection, segmentID)
|
||||
|
@ -155,13 +155,13 @@ func (container *ColSegContainer) addSegment(collection *Collection, partition *
|
|||
for _, p := range *col.Partitions() {
|
||||
if p.Tag() == partition.Tag() {
|
||||
*p.Segments() = append(*p.Segments(), newSegment)
|
||||
return newSegment, nil
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.New("cannot find collection or segment")
|
||||
return errors.New("cannot find collection or segment")
|
||||
}
|
||||
|
||||
func (container *ColSegContainer) removeSegment(segment *Segment) error {
|
||||
|
|
|
@ -1,675 +0,0 @@
|
|||
package reader
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
|
||||
"testing"
|
||||
)
|
||||
|
||||
//----------------------------------------------------------------------------------------------------- collection
|
||||
func TestColSegContainer_addCollection(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pulsarUrl := "pulsar://localhost:6650"
|
||||
node := NewQueryNode(ctx, 0, pulsarUrl)
|
||||
|
||||
fieldVec := schemapb.FieldSchema{
|
||||
Name: "vec",
|
||||
DataType: schemapb.DataType_VECTOR_FLOAT,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "16",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
fieldInt := schemapb.FieldSchema{
|
||||
Name: "age",
|
||||
DataType: schemapb.DataType_INT32,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
schema := schemapb.CollectionSchema{
|
||||
Name: "collection0",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
&fieldVec, &fieldInt,
|
||||
},
|
||||
}
|
||||
|
||||
collectionMeta := etcdpb.CollectionMeta{
|
||||
Id: UniqueID(0),
|
||||
Schema: &schema,
|
||||
CreateTime: Timestamp(0),
|
||||
SegmentIds: []UniqueID{0},
|
||||
PartitionTags: []string{"default"},
|
||||
}
|
||||
|
||||
collectionMetaBlob := proto.MarshalTextString(&collectionMeta)
|
||||
assert.NotEqual(t, "", collectionMetaBlob)
|
||||
|
||||
var collection = node.container.addCollection(&collectionMeta, collectionMetaBlob)
|
||||
|
||||
assert.Equal(t, collection.meta.Schema.Name, "collection0")
|
||||
assert.Equal(t, collection.meta.Id, UniqueID(0))
|
||||
assert.Equal(t, len(node.container.collections), 1)
|
||||
}
|
||||
|
||||
func TestColSegContainer_removeCollection(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pulsarUrl := "pulsar://localhost:6650"
|
||||
node := NewQueryNode(ctx, 0, pulsarUrl)
|
||||
|
||||
fieldVec := schemapb.FieldSchema{
|
||||
Name: "vec",
|
||||
DataType: schemapb.DataType_VECTOR_FLOAT,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "16",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
fieldInt := schemapb.FieldSchema{
|
||||
Name: "age",
|
||||
DataType: schemapb.DataType_INT32,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
schema := schemapb.CollectionSchema{
|
||||
Name: "collection0",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
&fieldVec, &fieldInt,
|
||||
},
|
||||
}
|
||||
|
||||
collectionMeta := etcdpb.CollectionMeta{
|
||||
Id: UniqueID(0),
|
||||
Schema: &schema,
|
||||
CreateTime: Timestamp(0),
|
||||
SegmentIds: []UniqueID{0},
|
||||
PartitionTags: []string{"default"},
|
||||
}
|
||||
|
||||
collectionMetaBlob := proto.MarshalTextString(&collectionMeta)
|
||||
assert.NotEqual(t, "", collectionMetaBlob)
|
||||
|
||||
var collection = node.container.addCollection(&collectionMeta, collectionMetaBlob)
|
||||
|
||||
assert.Equal(t, collection.meta.Schema.Name, "collection0")
|
||||
assert.Equal(t, collection.meta.Id, UniqueID(0))
|
||||
assert.Equal(t, len(node.container.collections), 1)
|
||||
|
||||
err := node.container.removeCollection(collection)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, len(node.container.collections), 0)
|
||||
}
|
||||
|
||||
func TestColSegContainer_getCollectionByID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pulsarUrl := "pulsar://localhost:6650"
|
||||
node := NewQueryNode(ctx, 0, pulsarUrl)
|
||||
|
||||
fieldVec := schemapb.FieldSchema{
|
||||
Name: "vec",
|
||||
DataType: schemapb.DataType_VECTOR_FLOAT,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "16",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
fieldInt := schemapb.FieldSchema{
|
||||
Name: "age",
|
||||
DataType: schemapb.DataType_INT32,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
schema := schemapb.CollectionSchema{
|
||||
Name: "collection0",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
&fieldVec, &fieldInt,
|
||||
},
|
||||
}
|
||||
|
||||
collectionMeta := etcdpb.CollectionMeta{
|
||||
Id: UniqueID(0),
|
||||
Schema: &schema,
|
||||
CreateTime: Timestamp(0),
|
||||
SegmentIds: []UniqueID{0},
|
||||
PartitionTags: []string{"default"},
|
||||
}
|
||||
|
||||
collectionMetaBlob := proto.MarshalTextString(&collectionMeta)
|
||||
assert.NotEqual(t, "", collectionMetaBlob)
|
||||
|
||||
var collection = node.container.addCollection(&collectionMeta, collectionMetaBlob)
|
||||
|
||||
assert.Equal(t, collection.meta.Schema.Name, "collection0")
|
||||
assert.Equal(t, collection.meta.Id, UniqueID(0))
|
||||
assert.Equal(t, len(node.container.collections), 1)
|
||||
|
||||
targetCollection, err := node.container.getCollectionByID(UniqueID(0))
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, targetCollection)
|
||||
assert.Equal(t, targetCollection.meta.Schema.Name, "collection0")
|
||||
assert.Equal(t, targetCollection.meta.Id, UniqueID(0))
|
||||
}
|
||||
|
||||
func TestColSegContainer_getCollectionByName(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pulsarUrl := "pulsar://localhost:6650"
|
||||
node := NewQueryNode(ctx, 0, pulsarUrl)
|
||||
|
||||
fieldVec := schemapb.FieldSchema{
|
||||
Name: "vec",
|
||||
DataType: schemapb.DataType_VECTOR_FLOAT,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "16",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
fieldInt := schemapb.FieldSchema{
|
||||
Name: "age",
|
||||
DataType: schemapb.DataType_INT32,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
schema := schemapb.CollectionSchema{
|
||||
Name: "collection0",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
&fieldVec, &fieldInt,
|
||||
},
|
||||
}
|
||||
|
||||
collectionMeta := etcdpb.CollectionMeta{
|
||||
Id: UniqueID(0),
|
||||
Schema: &schema,
|
||||
CreateTime: Timestamp(0),
|
||||
SegmentIds: []UniqueID{0},
|
||||
PartitionTags: []string{"default"},
|
||||
}
|
||||
|
||||
collectionMetaBlob := proto.MarshalTextString(&collectionMeta)
|
||||
assert.NotEqual(t, "", collectionMetaBlob)
|
||||
|
||||
var collection = node.container.addCollection(&collectionMeta, collectionMetaBlob)
|
||||
|
||||
assert.Equal(t, collection.meta.Schema.Name, "collection0")
|
||||
assert.Equal(t, collection.meta.Id, UniqueID(0))
|
||||
assert.Equal(t, len(node.container.collections), 1)
|
||||
|
||||
targetCollection, err := node.container.getCollectionByName("collection0")
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, targetCollection)
|
||||
assert.Equal(t, targetCollection.meta.Schema.Name, "collection0")
|
||||
assert.Equal(t, targetCollection.meta.Id, UniqueID(0))
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------------------------------- partition
|
||||
func TestColSegContainer_addPartition(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pulsarUrl := "pulsar://localhost:6650"
|
||||
node := NewQueryNode(ctx, 0, pulsarUrl)
|
||||
|
||||
fieldVec := schemapb.FieldSchema{
|
||||
Name: "vec",
|
||||
DataType: schemapb.DataType_VECTOR_FLOAT,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "16",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
fieldInt := schemapb.FieldSchema{
|
||||
Name: "age",
|
||||
DataType: schemapb.DataType_INT32,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
schema := schemapb.CollectionSchema{
|
||||
Name: "collection0",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
&fieldVec, &fieldInt,
|
||||
},
|
||||
}
|
||||
|
||||
collectionMeta := etcdpb.CollectionMeta{
|
||||
Id: UniqueID(0),
|
||||
Schema: &schema,
|
||||
CreateTime: Timestamp(0),
|
||||
SegmentIds: []UniqueID{0},
|
||||
PartitionTags: []string{"default"},
|
||||
}
|
||||
|
||||
collectionMetaBlob := proto.MarshalTextString(&collectionMeta)
|
||||
assert.NotEqual(t, "", collectionMetaBlob)
|
||||
|
||||
var collection = node.container.addCollection(&collectionMeta, collectionMetaBlob)
|
||||
|
||||
assert.Equal(t, collection.meta.Schema.Name, "collection0")
|
||||
assert.Equal(t, collection.meta.Id, UniqueID(0))
|
||||
assert.Equal(t, len(node.container.collections), 1)
|
||||
|
||||
for _, tag := range collectionMeta.PartitionTags {
|
||||
targetPartition, err := node.container.addPartition(collection, tag)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, targetPartition.partitionTag, "default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestColSegContainer_removePartition(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pulsarUrl := "pulsar://localhost:6650"
|
||||
node := NewQueryNode(ctx, 0, pulsarUrl)
|
||||
|
||||
fieldVec := schemapb.FieldSchema{
|
||||
Name: "vec",
|
||||
DataType: schemapb.DataType_VECTOR_FLOAT,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "16",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
fieldInt := schemapb.FieldSchema{
|
||||
Name: "age",
|
||||
DataType: schemapb.DataType_INT32,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
schema := schemapb.CollectionSchema{
|
||||
Name: "collection0",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
&fieldVec, &fieldInt,
|
||||
},
|
||||
}
|
||||
|
||||
collectionMeta := etcdpb.CollectionMeta{
|
||||
Id: UniqueID(0),
|
||||
Schema: &schema,
|
||||
CreateTime: Timestamp(0),
|
||||
SegmentIds: []UniqueID{0},
|
||||
PartitionTags: []string{"default"},
|
||||
}
|
||||
|
||||
collectionMetaBlob := proto.MarshalTextString(&collectionMeta)
|
||||
assert.NotEqual(t, "", collectionMetaBlob)
|
||||
|
||||
var collection = node.container.addCollection(&collectionMeta, collectionMetaBlob)
|
||||
|
||||
assert.Equal(t, collection.meta.Schema.Name, "collection0")
|
||||
assert.Equal(t, collection.meta.Id, UniqueID(0))
|
||||
assert.Equal(t, len(node.container.collections), 1)
|
||||
|
||||
for _, tag := range collectionMeta.PartitionTags {
|
||||
targetPartition, err := node.container.addPartition(collection, tag)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, targetPartition.partitionTag, "default")
|
||||
err = node.container.removePartition(targetPartition)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestColSegContainer_getPartitionByTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pulsarUrl := "pulsar://localhost:6650"
|
||||
node := NewQueryNode(ctx, 0, pulsarUrl)
|
||||
|
||||
fieldVec := schemapb.FieldSchema{
|
||||
Name: "vec",
|
||||
DataType: schemapb.DataType_VECTOR_FLOAT,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "16",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
fieldInt := schemapb.FieldSchema{
|
||||
Name: "age",
|
||||
DataType: schemapb.DataType_INT32,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
schema := schemapb.CollectionSchema{
|
||||
Name: "collection0",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
&fieldVec, &fieldInt,
|
||||
},
|
||||
}
|
||||
|
||||
collectionMeta := etcdpb.CollectionMeta{
|
||||
Id: UniqueID(0),
|
||||
Schema: &schema,
|
||||
CreateTime: Timestamp(0),
|
||||
SegmentIds: []UniqueID{0},
|
||||
PartitionTags: []string{"default"},
|
||||
}
|
||||
|
||||
collectionMetaBlob := proto.MarshalTextString(&collectionMeta)
|
||||
assert.NotEqual(t, "", collectionMetaBlob)
|
||||
|
||||
var collection = node.container.addCollection(&collectionMeta, collectionMetaBlob)
|
||||
|
||||
assert.Equal(t, collection.meta.Schema.Name, "collection0")
|
||||
assert.Equal(t, collection.meta.Id, UniqueID(0))
|
||||
assert.Equal(t, len(node.container.collections), 1)
|
||||
|
||||
for _, tag := range collectionMeta.PartitionTags {
|
||||
targetPartition, err := node.container.addPartition(collection, tag)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, targetPartition.partitionTag, "default")
|
||||
partition, err := node.container.getPartitionByTag(tag)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, partition)
|
||||
assert.Equal(t, partition.partitionTag, "default")
|
||||
}
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------------------------------- segment
|
||||
func TestColSegContainer_addSegment(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pulsarUrl := "pulsar://localhost:6650"
|
||||
node := NewQueryNode(ctx, 0, pulsarUrl)
|
||||
|
||||
fieldVec := schemapb.FieldSchema{
|
||||
Name: "vec",
|
||||
DataType: schemapb.DataType_VECTOR_FLOAT,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "16",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
fieldInt := schemapb.FieldSchema{
|
||||
Name: "age",
|
||||
DataType: schemapb.DataType_INT32,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
schema := schemapb.CollectionSchema{
|
||||
Name: "collection0",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
&fieldVec, &fieldInt,
|
||||
},
|
||||
}
|
||||
|
||||
collectionMeta := etcdpb.CollectionMeta{
|
||||
Id: UniqueID(0),
|
||||
Schema: &schema,
|
||||
CreateTime: Timestamp(0),
|
||||
SegmentIds: []UniqueID{0},
|
||||
PartitionTags: []string{"default"},
|
||||
}
|
||||
|
||||
collectionMetaBlob := proto.MarshalTextString(&collectionMeta)
|
||||
assert.NotEqual(t, "", collectionMetaBlob)
|
||||
|
||||
var collection = node.container.addCollection(&collectionMeta, collectionMetaBlob)
|
||||
|
||||
assert.Equal(t, collection.meta.Schema.Name, "collection0")
|
||||
assert.Equal(t, collection.meta.Id, UniqueID(0))
|
||||
assert.Equal(t, len(node.container.collections), 1)
|
||||
|
||||
partition, err := node.container.addPartition(collection, collectionMeta.PartitionTags[0])
|
||||
assert.NoError(t, err)
|
||||
|
||||
const segmentNum = 3
|
||||
for i := 0; i < segmentNum; i++ {
|
||||
targetSeg, err := node.container.addSegment(collection, partition, UniqueID(i))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, targetSeg.segmentID, UniqueID(i))
|
||||
}
|
||||
}
|
||||
|
||||
func TestColSegContainer_removeSegment(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pulsarUrl := "pulsar://localhost:6650"
|
||||
node := NewQueryNode(ctx, 0, pulsarUrl)
|
||||
|
||||
fieldVec := schemapb.FieldSchema{
|
||||
Name: "vec",
|
||||
DataType: schemapb.DataType_VECTOR_FLOAT,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "16",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
fieldInt := schemapb.FieldSchema{
|
||||
Name: "age",
|
||||
DataType: schemapb.DataType_INT32,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
schema := schemapb.CollectionSchema{
|
||||
Name: "collection0",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
&fieldVec, &fieldInt,
|
||||
},
|
||||
}
|
||||
|
||||
collectionMeta := etcdpb.CollectionMeta{
|
||||
Id: UniqueID(0),
|
||||
Schema: &schema,
|
||||
CreateTime: Timestamp(0),
|
||||
SegmentIds: []UniqueID{0},
|
||||
PartitionTags: []string{"default"},
|
||||
}
|
||||
|
||||
collectionMetaBlob := proto.MarshalTextString(&collectionMeta)
|
||||
assert.NotEqual(t, "", collectionMetaBlob)
|
||||
|
||||
var collection = node.container.addCollection(&collectionMeta, collectionMetaBlob)
|
||||
|
||||
assert.Equal(t, collection.meta.Schema.Name, "collection0")
|
||||
assert.Equal(t, collection.meta.Id, UniqueID(0))
|
||||
assert.Equal(t, len(node.container.collections), 1)
|
||||
|
||||
partition, err := node.container.addPartition(collection, collectionMeta.PartitionTags[0])
|
||||
assert.NoError(t, err)
|
||||
|
||||
const segmentNum = 3
|
||||
for i := 0; i < segmentNum; i++ {
|
||||
targetSeg, err := node.container.addSegment(collection, partition, UniqueID(i))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, targetSeg.segmentID, UniqueID(i))
|
||||
err = node.container.removeSegment(targetSeg)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestColSegContainer_getSegmentByID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pulsarUrl := "pulsar://localhost:6650"
|
||||
node := NewQueryNode(ctx, 0, pulsarUrl)
|
||||
|
||||
fieldVec := schemapb.FieldSchema{
|
||||
Name: "vec",
|
||||
DataType: schemapb.DataType_VECTOR_FLOAT,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "16",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
fieldInt := schemapb.FieldSchema{
|
||||
Name: "age",
|
||||
DataType: schemapb.DataType_INT32,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
schema := schemapb.CollectionSchema{
|
||||
Name: "collection0",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
&fieldVec, &fieldInt,
|
||||
},
|
||||
}
|
||||
|
||||
collectionMeta := etcdpb.CollectionMeta{
|
||||
Id: UniqueID(0),
|
||||
Schema: &schema,
|
||||
CreateTime: Timestamp(0),
|
||||
SegmentIds: []UniqueID{0},
|
||||
PartitionTags: []string{"default"},
|
||||
}
|
||||
|
||||
collectionMetaBlob := proto.MarshalTextString(&collectionMeta)
|
||||
assert.NotEqual(t, "", collectionMetaBlob)
|
||||
|
||||
var collection = node.container.addCollection(&collectionMeta, collectionMetaBlob)
|
||||
|
||||
assert.Equal(t, collection.meta.Schema.Name, "collection0")
|
||||
assert.Equal(t, collection.meta.Id, UniqueID(0))
|
||||
assert.Equal(t, len(node.container.collections), 1)
|
||||
|
||||
partition, err := node.container.addPartition(collection, collectionMeta.PartitionTags[0])
|
||||
assert.NoError(t, err)
|
||||
|
||||
const segmentNum = 3
|
||||
for i := 0; i < segmentNum; i++ {
|
||||
targetSeg, err := node.container.addSegment(collection, partition, UniqueID(i))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, targetSeg.segmentID, UniqueID(i))
|
||||
seg, err := node.container.getSegmentByID(UniqueID(i))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, seg.segmentID, UniqueID(i))
|
||||
}
|
||||
}
|
||||
|
||||
func TestColSegContainer_hasSegment(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pulsarUrl := "pulsar://localhost:6650"
|
||||
node := NewQueryNode(ctx, 0, pulsarUrl)
|
||||
|
||||
fieldVec := schemapb.FieldSchema{
|
||||
Name: "vec",
|
||||
DataType: schemapb.DataType_VECTOR_FLOAT,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "16",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
fieldInt := schemapb.FieldSchema{
|
||||
Name: "age",
|
||||
DataType: schemapb.DataType_INT32,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
schema := schemapb.CollectionSchema{
|
||||
Name: "collection0",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
&fieldVec, &fieldInt,
|
||||
},
|
||||
}
|
||||
|
||||
collectionMeta := etcdpb.CollectionMeta{
|
||||
Id: UniqueID(0),
|
||||
Schema: &schema,
|
||||
CreateTime: Timestamp(0),
|
||||
SegmentIds: []UniqueID{0},
|
||||
PartitionTags: []string{"default"},
|
||||
}
|
||||
|
||||
collectionMetaBlob := proto.MarshalTextString(&collectionMeta)
|
||||
assert.NotEqual(t, "", collectionMetaBlob)
|
||||
|
||||
var collection = node.container.addCollection(&collectionMeta, collectionMetaBlob)
|
||||
|
||||
assert.Equal(t, collection.meta.Schema.Name, "collection0")
|
||||
assert.Equal(t, collection.meta.Id, UniqueID(0))
|
||||
assert.Equal(t, len(node.container.collections), 1)
|
||||
|
||||
partition, err := node.container.addPartition(collection, collectionMeta.PartitionTags[0])
|
||||
assert.NoError(t, err)
|
||||
|
||||
const segmentNum = 3
|
||||
for i := 0; i < segmentNum; i++ {
|
||||
targetSeg, err := node.container.addSegment(collection, partition, UniqueID(i))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, targetSeg.segmentID, UniqueID(i))
|
||||
hasSeg := node.container.hasSegment(UniqueID(i))
|
||||
assert.Equal(t, hasSeg, true)
|
||||
hasSeg = node.container.hasSegment(UniqueID(i + 100))
|
||||
assert.Equal(t, hasSeg, false)
|
||||
}
|
||||
}
|
|
@ -1,165 +1,33 @@
|
|||
package reader
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCollection_Partitions(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pulsarUrl := "pulsar://localhost:6650"
|
||||
node := NewQueryNode(ctx, 0, pulsarUrl)
|
||||
|
||||
fieldVec := schemapb.FieldSchema{
|
||||
Name: "vec",
|
||||
DataType: schemapb.DataType_VECTOR_FLOAT,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "16",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
fieldInt := schemapb.FieldSchema{
|
||||
Name: "age",
|
||||
DataType: schemapb.DataType_INT32,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
schema := schemapb.CollectionSchema{
|
||||
Name: "collection0",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
&fieldVec, &fieldInt,
|
||||
},
|
||||
}
|
||||
|
||||
collectionMeta := etcdpb.CollectionMeta{
|
||||
Id: UniqueID(0),
|
||||
Schema: &schema,
|
||||
CreateTime: Timestamp(0),
|
||||
SegmentIds: []UniqueID{0},
|
||||
PartitionTags: []string{"default"},
|
||||
}
|
||||
|
||||
collectionMetaBlob := proto.MarshalTextString(&collectionMeta)
|
||||
assert.NotEqual(t, "", collectionMetaBlob)
|
||||
|
||||
var collection = node.container.addCollection(&collectionMeta, collectionMetaBlob)
|
||||
|
||||
assert.Equal(t, collection.meta.Schema.Name, "collection0")
|
||||
assert.Equal(t, collection.meta.Id, UniqueID(0))
|
||||
assert.Equal(t, len(node.container.collections), 1)
|
||||
|
||||
for _, tag := range collectionMeta.PartitionTags {
|
||||
_, err := node.container.addPartition(collection, tag)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
partitions := collection.Partitions()
|
||||
assert.Equal(t, len(collectionMeta.PartitionTags), len(*partitions))
|
||||
}
|
||||
|
||||
func TestCollection_newCollection(t *testing.T) {
|
||||
fieldVec := schemapb.FieldSchema{
|
||||
Name: "vec",
|
||||
DataType: schemapb.DataType_VECTOR_FLOAT,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "16",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
fieldInt := schemapb.FieldSchema{
|
||||
Name: "age",
|
||||
DataType: schemapb.DataType_INT32,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
schema := schemapb.CollectionSchema{
|
||||
Name: "collection0",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
&fieldVec, &fieldInt,
|
||||
},
|
||||
}
|
||||
|
||||
collectionMeta := etcdpb.CollectionMeta{
|
||||
Id: UniqueID(0),
|
||||
Schema: &schema,
|
||||
CreateTime: Timestamp(0),
|
||||
SegmentIds: []UniqueID{0},
|
||||
PartitionTags: []string{"default"},
|
||||
}
|
||||
|
||||
collectionMetaBlob := proto.MarshalTextString(&collectionMeta)
|
||||
assert.NotEqual(t, "", collectionMetaBlob)
|
||||
|
||||
collection := newCollection(&collectionMeta, collectionMetaBlob)
|
||||
assert.Equal(t, collection.meta.Schema.Name, "collection0")
|
||||
assert.Equal(t, collection.meta.Id, UniqueID(0))
|
||||
}
|
||||
|
||||
func TestCollection_deleteCollection(t *testing.T) {
|
||||
fieldVec := schemapb.FieldSchema{
|
||||
Name: "vec",
|
||||
DataType: schemapb.DataType_VECTOR_FLOAT,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "16",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
fieldInt := schemapb.FieldSchema{
|
||||
Name: "age",
|
||||
DataType: schemapb.DataType_INT32,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
schema := schemapb.CollectionSchema{
|
||||
Name: "collection0",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
&fieldVec, &fieldInt,
|
||||
},
|
||||
}
|
||||
|
||||
collectionMeta := etcdpb.CollectionMeta{
|
||||
Id: UniqueID(0),
|
||||
Schema: &schema,
|
||||
CreateTime: Timestamp(0),
|
||||
SegmentIds: []UniqueID{0},
|
||||
PartitionTags: []string{"default"},
|
||||
}
|
||||
|
||||
collectionMetaBlob := proto.MarshalTextString(&collectionMeta)
|
||||
assert.NotEqual(t, "", collectionMetaBlob)
|
||||
|
||||
collection := newCollection(&collectionMeta, collectionMetaBlob)
|
||||
assert.Equal(t, collection.meta.Schema.Name, "collection0")
|
||||
assert.Equal(t, collection.meta.Id, UniqueID(0))
|
||||
|
||||
deleteCollection(collection)
|
||||
}
|
||||
//func TestCollection_NewPartition(t *testing.T) {
|
||||
// ctx := context.Background()
|
||||
// pulsarUrl := "pulsar://localhost:6650"
|
||||
// node := NewQueryNode(ctx, 0, pulsarUrl)
|
||||
//
|
||||
// var collection = node.newCollection(0, "collection0", "")
|
||||
// var partition = collection.newPartition("partition0")
|
||||
//
|
||||
// assert.Equal(t, collection.CollectionName, "collection0")
|
||||
// assert.Equal(t, collection.CollectionID, int64(0))
|
||||
// assert.Equal(t, partition.partitionTag, "partition0")
|
||||
// assert.Equal(t, len(collection.Partitions), 1)
|
||||
//}
|
||||
//
|
||||
//func TestCollection_DeletePartition(t *testing.T) {
|
||||
// ctx := context.Background()
|
||||
// pulsarUrl := "pulsar://localhost:6650"
|
||||
// node := NewQueryNode(ctx, 0, pulsarUrl)
|
||||
//
|
||||
// var collection = node.newCollection(0, "collection0", "")
|
||||
// var partition = collection.newPartition("partition0")
|
||||
//
|
||||
// assert.Equal(t, collection.CollectionName, "collection0")
|
||||
// assert.Equal(t, collection.CollectionID, int64(0))
|
||||
// assert.Equal(t, partition.partitionTag, "partition0")
|
||||
// assert.Equal(t, len(collection.Partitions), 1)
|
||||
//
|
||||
// collection.deletePartition(node, partition)
|
||||
//
|
||||
// assert.Equal(t, len(collection.Partitions), 0)
|
||||
//}
|
||||
|
|
|
@ -3,7 +3,7 @@ package reader
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/gogo/protobuf/proto"
|
||||
"log"
|
||||
"path"
|
||||
"reflect"
|
||||
|
@ -144,7 +144,7 @@ func (mService *metaService) processCollectionCreate(id string, value string) {
|
|||
if col != nil {
|
||||
newCollection := mService.container.addCollection(col, value)
|
||||
for _, partitionTag := range col.PartitionTags {
|
||||
_, err := mService.container.addPartition(newCollection, partitionTag)
|
||||
err := mService.container.addPartition(newCollection, partitionTag)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
@ -174,7 +174,7 @@ func (mService *metaService) processSegmentCreate(id string, value string) {
|
|||
return
|
||||
}
|
||||
if partition != nil {
|
||||
_, err = mService.container.addSegment(col, partition, seg.SegmentId)
|
||||
err = mService.container.addSegment(col, partition, seg.SegmentId)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return
|
||||
|
|
|
@ -1,88 +1,57 @@
|
|||
package reader
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPartition_Segments(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pulsarUrl := "pulsar://localhost:6650"
|
||||
node := NewQueryNode(ctx, 0, pulsarUrl)
|
||||
|
||||
fieldVec := schemapb.FieldSchema{
|
||||
Name: "vec",
|
||||
DataType: schemapb.DataType_VECTOR_FLOAT,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "16",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
fieldInt := schemapb.FieldSchema{
|
||||
Name: "age",
|
||||
DataType: schemapb.DataType_INT32,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
schema := schemapb.CollectionSchema{
|
||||
Name: "collection0",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
&fieldVec, &fieldInt,
|
||||
},
|
||||
}
|
||||
|
||||
collectionMeta := etcdpb.CollectionMeta{
|
||||
Id: UniqueID(0),
|
||||
Schema: &schema,
|
||||
CreateTime: Timestamp(0),
|
||||
SegmentIds: []UniqueID{0},
|
||||
PartitionTags: []string{"default"},
|
||||
}
|
||||
|
||||
collectionMetaBlob := proto.MarshalTextString(&collectionMeta)
|
||||
assert.NotEqual(t, "", collectionMetaBlob)
|
||||
|
||||
var collection = node.container.addCollection(&collectionMeta, collectionMetaBlob)
|
||||
|
||||
assert.Equal(t, collection.meta.Schema.Name, "collection0")
|
||||
assert.Equal(t, collection.meta.Id, UniqueID(0))
|
||||
assert.Equal(t, len(node.container.collections), 1)
|
||||
|
||||
for _, tag := range collectionMeta.PartitionTags {
|
||||
_, err := node.container.addPartition(collection, tag)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
partitions := collection.Partitions()
|
||||
assert.Equal(t, len(collectionMeta.PartitionTags), len(*partitions))
|
||||
|
||||
targetPartition := (*partitions)[0]
|
||||
|
||||
const segmentNum = 3
|
||||
for i:= 0; i < segmentNum; i++ {
|
||||
_, err := node.container.addSegment(collection, targetPartition, UniqueID(i))
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
segments := targetPartition.Segments()
|
||||
assert.Equal(t, segmentNum, len(*segments))
|
||||
}
|
||||
|
||||
func TestPartition_newPartition(t *testing.T) {
|
||||
partitionTag := "default"
|
||||
partition := newPartition(partitionTag)
|
||||
assert.Equal(t, partition.partitionTag, partitionTag)
|
||||
}
|
||||
//func TestPartition_NewSegment(t *testing.T) {
|
||||
// ctx := context.Background()
|
||||
// pulsarUrl := "pulsar://localhost:6650"
|
||||
// node := NewQueryNode(ctx, 0, pulsarUrl)
|
||||
//
|
||||
// var collection = node.newCollection(0, "collection0", "")
|
||||
// var partition = collection.newPartition("partition0")
|
||||
//
|
||||
// var segment = partition.newSegment(0)
|
||||
// node.SegmentsMap[int64(0)] = segment
|
||||
//
|
||||
// assert.Equal(t, collection.CollectionName, "collection0")
|
||||
// assert.Equal(t, collection.CollectionID, int64(0))
|
||||
// assert.Equal(t, partition.partitionTag, "partition0")
|
||||
// assert.Equal(t, node.Collections[0].Partitions[0].Segments[0].SegmentID, int64(0))
|
||||
//
|
||||
// assert.Equal(t, len(collection.Partitions), 1)
|
||||
// assert.Equal(t, len(node.Collections), 1)
|
||||
// assert.Equal(t, len(node.Collections[0].Partitions[0].Segments), 1)
|
||||
//
|
||||
// assert.Equal(t, segment.SegmentID, int64(0))
|
||||
// assert.Equal(t, node.foundSegmentBySegmentID(int64(0)), true)
|
||||
//}
|
||||
//
|
||||
//func TestPartition_DeleteSegment(t *testing.T) {
|
||||
// // 1. Construct node, collection, partition and segment
|
||||
// ctx := context.Background()
|
||||
// pulsarUrl := "pulsar://localhost:6650"
|
||||
// node := NewQueryNode(ctx, 0, pulsarUrl)
|
||||
//
|
||||
// var collection = node.newCollection(0, "collection0", "")
|
||||
// var partition = collection.newPartition("partition0")
|
||||
//
|
||||
// var segment = partition.newSegment(0)
|
||||
// node.SegmentsMap[int64(0)] = segment
|
||||
//
|
||||
// assert.Equal(t, collection.CollectionName, "collection0")
|
||||
// assert.Equal(t, collection.CollectionID, int64(0))
|
||||
// assert.Equal(t, partition.partitionTag, "partition0")
|
||||
// assert.Equal(t, node.Collections[0].Partitions[0].Segments[0].SegmentID, int64(0))
|
||||
//
|
||||
// assert.Equal(t, len(collection.Partitions), 1)
|
||||
// assert.Equal(t, len(node.Collections), 1)
|
||||
// assert.Equal(t, len(node.Collections[0].Partitions[0].Segments), 1)
|
||||
//
|
||||
// assert.Equal(t, segment.SegmentID, int64(0))
|
||||
//
|
||||
// // 2. Destruct collection, partition and segment
|
||||
// partition.deleteSegment(node, segment)
|
||||
//
|
||||
// assert.Equal(t, len(collection.Partitions), 1)
|
||||
// assert.Equal(t, len(node.Collections), 1)
|
||||
// assert.Equal(t, len(node.Collections[0].Partitions[0].Segments), 0)
|
||||
// assert.Equal(t, node.foundSegmentBySegmentID(int64(0)), false)
|
||||
//}
|
||||
|
|
|
@ -84,7 +84,7 @@ func (s *Segment) getMemSize() int64 {
|
|||
return int64(memoryUsageInBytes)
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------------------------- preDm functions
|
||||
//-------------------------------------------------------------------------------------- preprocess functions
|
||||
func (s *Segment) segmentPreInsert(numOfRecords int) int64 {
|
||||
/*
|
||||
long int
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -1,14 +1,7 @@
|
|||
package tsoutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv"
|
||||
"go.etcd.io/etcd/clientv3"
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -27,15 +20,3 @@ func ParseTS(ts uint64) (time.Time, uint64) {
|
|||
physicalTime := time.Unix(int64(physical/1000), int64(physical)%1000*time.Millisecond.Nanoseconds())
|
||||
return physicalTime, logical
|
||||
}
|
||||
|
||||
func NewTSOKVBase(subPath string) *kv.EtcdKV {
|
||||
etcdAddr := conf.Config.Etcd.Address
|
||||
etcdAddr += ":"
|
||||
etcdAddr += strconv.FormatInt(int64(conf.Config.Etcd.Port), 10)
|
||||
fmt.Println("etcdAddr ::: ", etcdAddr)
|
||||
client, _ := clientv3.New(clientv3.Config{
|
||||
Endpoints: []string{etcdAddr},
|
||||
DialTimeout: 5 * time.Second,
|
||||
})
|
||||
return kv.NewEtcdKV(client, path.Join(conf.Config.Etcd.Rootpath, subPath))
|
||||
}
|
||||
|
|
|
@ -37,7 +37,7 @@
|
|||
cd milvus-distributed
|
||||
pwd_dir=`pwd`
|
||||
export PATH=$PATH:$(go env GOPATH)/bin
|
||||
export protoc=${pwd_dir}/internal/core/cmake_build/thirdparty/protobuf/protobuf-build/protoc
|
||||
export protoc=${pwd_dir}/cmake_build/thirdparty/protobuf/protobuf-build/protoc
|
||||
./ci/scripts/proto_gen_go.sh
|
||||
```
|
||||
|
||||
|
|
Loading…
Reference in New Issue