milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: [GoSDK] Use generic column base and add unit tests (#37768) 

Related to #31293

Previous column is written using go before 1.18, which does not support
generic type. After migrating client pkg to main repo, generic base
could be used to reduce lots of duplicated code.

---------

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
pull/37794/head
congqixia 2024-11-19 10:40:31 +08:00 committed by GitHub
parent c79fbd5eab
commit 175f49ea60
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
23 changed files with 1894 additions and 3920 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

241
client/column/array.go
View File

@ -17,62 +17,18 @@
package column
import (
"fmt"
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/client/v2/entity"
)
// ColumnVarCharArray generated columns type for VarChar
type ColumnVarCharArray struct {
ColumnBase
name string
values [][][]byte
// columnArrayBase implement `Column` interface
// it provided specified `FieldData` behavior for Array columns.
type columnArrayBase[T any] struct {
*genericColumnBase[[]T]
elementType entity.FieldType
}
// Name returns column name
func (c *ColumnVarCharArray) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnVarCharArray) Type() entity.FieldType {
return entity.FieldTypeArray
}
// Len returns column values length
func (c *ColumnVarCharArray) Len() int {
return len(c.values)
}
func (c *ColumnVarCharArray) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &ColumnVarCharArray{
ColumnBase: c.ColumnBase,
name: c.name,
values: c.values[start:end],
}
}
// Get returns value at index as interface{}.
func (c *ColumnVarCharArray) Get(idx int) (interface{}, error) {
var r []string // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnVarCharArray) FieldData() *schemapb.FieldData {
func (c *columnArrayBase[T]) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_Array,
FieldName: c.name,
@ -80,24 +36,15 @@ func (c *ColumnVarCharArray) FieldData() *schemapb.FieldData {
data := make([]*schemapb.ScalarField, 0, c.Len())
for _, arr := range c.values {
converted := make([]string, 0, c.Len())
for i := 0; i < len(arr); i++ {
converted = append(converted, string(arr[i]))
}
data = append(data, &schemapb.ScalarField{
Data: &schemapb.ScalarField_StringData{
StringData: &schemapb.StringArray{
Data: converted,
},
},
})
data = append(data, slice2Scalar(arr, c.elementType))
}
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_ArrayData{
ArrayData: &schemapb.ArrayArray{
Data: data,
ElementType: schemapb.DataType_VarChar,
ElementType: schemapb.DataType(c.elementType),
},
},
},
@ -105,36 +52,162 @@ func (c *ColumnVarCharArray) FieldData() *schemapb.FieldData {
return fd
}
// ValueByIdx returns value of the provided index
// error occurs when index out of range
func (c *ColumnVarCharArray) ValueByIdx(idx int) ([][]byte, error) {
var r [][]byte // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
func (c *columnArrayBase[T]) ElementType() entity.FieldType {
return c.elementType
}
func (c *columnArrayBase[T]) slice(start, end int) *columnArrayBase[T] {
return &columnArrayBase[T]{
genericColumnBase: c.genericColumnBase.slice(start, end),
elementType: c.elementType,
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnVarCharArray) AppendValue(i interface{}) error {
v, ok := i.([][]byte)
if !ok {
return fmt.Errorf("invalid type, expected []string, got %T", i)
func newArrayBase[T any](fieldName string, data [][]T, elementType entity.FieldType) *columnArrayBase[T] {
return &columnArrayBase[T]{
genericColumnBase: &genericColumnBase[[]T]{
name: fieldName,
fieldType: entity.FieldTypeArray,
values: data,
},
elementType: elementType,
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnVarCharArray) Data() [][][]byte {
return c.values
/* bool array */
type ColumnBoolArray struct {
*columnArrayBase[bool]
}
// NewColumnVarChar auto generated constructor
func NewColumnVarCharArray(name string, values [][][]byte) *ColumnVarCharArray {
func NewColumnBoolArray(fieldName string, data [][]bool) *ColumnBoolArray {
return &ColumnBoolArray{
columnArrayBase: newArrayBase[bool](fieldName, data, entity.FieldTypeBool),
}
}
func (c *ColumnBoolArray) Slice(start, end int) Column {
return &ColumnBoolArray{
columnArrayBase: c.columnArrayBase.slice(start, end),
}
}
/* int8 array */
type ColumnInt8Array struct {
*columnArrayBase[int8]
}
func NewColumnInt8Array(fieldName string, data [][]int8) *ColumnInt8Array {
return &ColumnInt8Array{
columnArrayBase: newArrayBase(fieldName, data, entity.FieldTypeInt8),
}
}
func (c *ColumnInt8Array) Slice(start, end int) Column {
return &ColumnInt8Array{
columnArrayBase: c.columnArrayBase.slice(start, end),
}
}
/* int16 array */
type ColumnInt16Array struct {
*columnArrayBase[int16]
}
func NewColumnInt16Array(fieldName string, data [][]int16) *ColumnInt16Array {
return &ColumnInt16Array{
columnArrayBase: newArrayBase(fieldName, data, entity.FieldTypeInt16),
}
}
func (c *ColumnInt16Array) Slice(start, end int) Column {
return &ColumnInt16Array{
columnArrayBase: c.columnArrayBase.slice(start, end),
}
}
/* int32 array */
type ColumnInt32Array struct {
*columnArrayBase[int32]
}
func NewColumnInt32Array(fieldName string, data [][]int32) *ColumnInt32Array {
return &ColumnInt32Array{
columnArrayBase: newArrayBase(fieldName, data, entity.FieldTypeInt32),
}
}
func (c *ColumnInt32Array) Slice(start, end int) Column {
return &ColumnInt32Array{
columnArrayBase: c.columnArrayBase.slice(start, end),
}
}
/* int64 array */
type ColumnInt64Array struct {
*columnArrayBase[int64]
}
func NewColumnInt64Array(fieldName string, data [][]int64) *ColumnInt64Array {
return &ColumnInt64Array{
columnArrayBase: newArrayBase(fieldName, data, entity.FieldTypeInt64),
}
}
func (c *ColumnInt64Array) Slice(start, end int) Column {
return &ColumnInt64Array{
columnArrayBase: c.columnArrayBase.slice(start, end),
}
}
/* float32 array */
type ColumnFloatArray struct {
*columnArrayBase[float32]
}
func NewColumnFloatArray(fieldName string, data [][]float32) *ColumnFloatArray {
return &ColumnFloatArray{
columnArrayBase: newArrayBase(fieldName, data, entity.FieldTypeFloat),
}
}
func (c *ColumnFloatArray) Slice(start, end int) Column {
return &ColumnFloatArray{
columnArrayBase: c.columnArrayBase.slice(start, end),
}
}
/* float64 array */
type ColumnDoubleArray struct {
*columnArrayBase[float64]
}
func NewColumnDoubleArray(fieldName string, data [][]float64) *ColumnDoubleArray {
return &ColumnDoubleArray{
columnArrayBase: newArrayBase(fieldName, data, entity.FieldTypeDouble),
}
}
func (c *ColumnDoubleArray) Slice(start, end int) Column {
return &ColumnDoubleArray{
columnArrayBase: c.columnArrayBase.slice(start, end),
}
}
/* varchar array */
type ColumnVarCharArray struct {
*columnArrayBase[string]
}
func NewColumnVarCharArray(fieldName string, data [][]string) *ColumnVarCharArray {
return &ColumnVarCharArray{
name: name,
values: values,
columnArrayBase: newArrayBase(fieldName, data, entity.FieldTypeVarChar),
}
}

813
client/column/array_gen.go
View File

@ -1,813 +0,0 @@
// Code generated by go generate; DO NOT EDIT
// This file is generated by go generate
package column
import (
"fmt"
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/client/v2/entity"
)
// ColumnBoolArray generated columns type for Bool
type ColumnBoolArray struct {
ColumnBase
name string
values [][]bool
}
// Name returns column name
func (c *ColumnBoolArray) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnBoolArray) Type() entity.FieldType {
return entity.FieldTypeArray
}
// Len returns column values length
func (c *ColumnBoolArray) Len() int {
return len(c.values)
}
func (c *ColumnBoolArray) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
if end == -1 || end > l {
end = l
}
return &ColumnBoolArray{
ColumnBase: c.ColumnBase,
name: c.name,
values: c.values[start:end],
}
}
// Get returns value at index as interface{}.
func (c *ColumnBoolArray) Get(idx int) (interface{}, error) {
var r []bool // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnBoolArray) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_Array,
FieldName: c.name,
}
data := make([]*schemapb.ScalarField, 0, c.Len())
for _, arr := range c.values {
converted := make([]bool, 0, c.Len())
for i := 0; i < len(arr); i++ {
converted = append(converted, bool(arr[i]))
}
data = append(data, &schemapb.ScalarField{
Data: &schemapb.ScalarField_BoolData{
BoolData: &schemapb.BoolArray{
Data: converted,
},
},
})
}
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_ArrayData{
ArrayData: &schemapb.ArrayArray{
Data: data,
ElementType: schemapb.DataType_Bool,
},
},
},
}
return fd
}
// ValueByIdx returns value of the provided index
// error occurs when index out of range
func (c *ColumnBoolArray) ValueByIdx(idx int) ([]bool, error) {
var r []bool // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnBoolArray) AppendValue(i interface{}) error {
v, ok := i.([]bool)
if !ok {
return fmt.Errorf("invalid type, expected []bool, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnBoolArray) Data() [][]bool {
return c.values
}
// NewColumnBool auto generated constructor
func NewColumnBoolArray(name string, values [][]bool) *ColumnBoolArray {
return &ColumnBoolArray{
name: name,
values: values,
}
}
// ColumnInt8Array generated columns type for Int8
type ColumnInt8Array struct {
ColumnBase
name string
values [][]int8
}
// Name returns column name
func (c *ColumnInt8Array) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnInt8Array) Type() entity.FieldType {
return entity.FieldTypeArray
}
// Len returns column values length
func (c *ColumnInt8Array) Len() int {
return len(c.values)
}
func (c *ColumnInt8Array) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &ColumnInt8Array{
ColumnBase: c.ColumnBase,
name: c.name,
values: c.values[start:end],
}
}
// Get returns value at index as interface{}.
func (c *ColumnInt8Array) Get(idx int) (interface{}, error) {
var r []int8 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnInt8Array) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_Array,
FieldName: c.name,
}
data := make([]*schemapb.ScalarField, 0, c.Len())
for _, arr := range c.values {
converted := make([]int32, 0, c.Len())
for i := 0; i < len(arr); i++ {
converted = append(converted, int32(arr[i]))
}
data = append(data, &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: converted,
},
},
})
}
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_ArrayData{
ArrayData: &schemapb.ArrayArray{
Data: data,
ElementType: schemapb.DataType_Int8,
},
},
},
}
return fd
}
// ValueByIdx returns value of the provided index
// error occurs when index out of range
func (c *ColumnInt8Array) ValueByIdx(idx int) ([]int8, error) {
var r []int8 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnInt8Array) AppendValue(i interface{}) error {
v, ok := i.([]int8)
if !ok {
return fmt.Errorf("invalid type, expected []int8, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnInt8Array) Data() [][]int8 {
return c.values
}
// NewColumnInt8 auto generated constructor
func NewColumnInt8Array(name string, values [][]int8) *ColumnInt8Array {
return &ColumnInt8Array{
name: name,
values: values,
}
}
// ColumnInt16Array generated columns type for Int16
type ColumnInt16Array struct {
ColumnBase
name string
values [][]int16
}
// Name returns column name
func (c *ColumnInt16Array) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnInt16Array) Type() entity.FieldType {
return entity.FieldTypeArray
}
// Len returns column values length
func (c *ColumnInt16Array) Len() int {
return len(c.values)
}
func (c *ColumnInt16Array) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &ColumnInt16Array{
ColumnBase: c.ColumnBase,
name: c.name,
values: c.values[start:end],
}
}
// Get returns value at index as interface{}.
func (c *ColumnInt16Array) Get(idx int) (interface{}, error) {
var r []int16 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnInt16Array) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_Array,
FieldName: c.name,
}
data := make([]*schemapb.ScalarField, 0, c.Len())
for _, arr := range c.values {
converted := make([]int32, 0, c.Len())
for i := 0; i < len(arr); i++ {
converted = append(converted, int32(arr[i]))
}
data = append(data, &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: converted,
},
},
})
}
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_ArrayData{
ArrayData: &schemapb.ArrayArray{
Data: data,
ElementType: schemapb.DataType_Int16,
},
},
},
}
return fd
}
// ValueByIdx returns value of the provided index
// error occurs when index out of range
func (c *ColumnInt16Array) ValueByIdx(idx int) ([]int16, error) {
var r []int16 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnInt16Array) AppendValue(i interface{}) error {
v, ok := i.([]int16)
if !ok {
return fmt.Errorf("invalid type, expected []int16, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnInt16Array) Data() [][]int16 {
return c.values
}
// NewColumnInt16 auto generated constructor
func NewColumnInt16Array(name string, values [][]int16) *ColumnInt16Array {
return &ColumnInt16Array{
name: name,
values: values,
}
}
// ColumnInt32Array generated columns type for Int32
type ColumnInt32Array struct {
ColumnBase
name string
values [][]int32
}
// Name returns column name
func (c *ColumnInt32Array) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnInt32Array) Type() entity.FieldType {
return entity.FieldTypeArray
}
// Len returns column values length
func (c *ColumnInt32Array) Len() int {
return len(c.values)
}
func (c *ColumnInt32Array) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &ColumnInt32Array{
ColumnBase: c.ColumnBase,
name: c.name,
values: c.values[start:end],
}
}
// Get returns value at index as interface{}.
func (c *ColumnInt32Array) Get(idx int) (interface{}, error) {
var r []int32 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnInt32Array) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_Array,
FieldName: c.name,
}
data := make([]*schemapb.ScalarField, 0, c.Len())
for _, arr := range c.values {
converted := make([]int32, 0, c.Len())
for i := 0; i < len(arr); i++ {
converted = append(converted, int32(arr[i]))
}
data = append(data, &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: converted,
},
},
})
}
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_ArrayData{
ArrayData: &schemapb.ArrayArray{
Data: data,
ElementType: schemapb.DataType_Int32,
},
},
},
}
return fd
}
// ValueByIdx returns value of the provided index
// error occurs when index out of range
func (c *ColumnInt32Array) ValueByIdx(idx int) ([]int32, error) {
var r []int32 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnInt32Array) AppendValue(i interface{}) error {
v, ok := i.([]int32)
if !ok {
return fmt.Errorf("invalid type, expected []int32, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnInt32Array) Data() [][]int32 {
return c.values
}
// NewColumnInt32 auto generated constructor
func NewColumnInt32Array(name string, values [][]int32) *ColumnInt32Array {
return &ColumnInt32Array{
name: name,
values: values,
}
}
// ColumnInt64Array generated columns type for Int64
type ColumnInt64Array struct {
ColumnBase
name string
values [][]int64
}
// Name returns column name
func (c *ColumnInt64Array) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnInt64Array) Type() entity.FieldType {
return entity.FieldTypeArray
}
// Len returns column values length
func (c *ColumnInt64Array) Len() int {
return len(c.values)
}
func (c *ColumnInt64Array) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &ColumnInt64Array{
ColumnBase: c.ColumnBase,
name: c.name,
values: c.values[start:end],
}
}
// Get returns value at index as interface{}.
func (c *ColumnInt64Array) Get(idx int) (interface{}, error) {
var r []int64 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnInt64Array) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_Array,
FieldName: c.name,
}
data := make([]*schemapb.ScalarField, 0, c.Len())
for _, arr := range c.values {
converted := make([]int64, 0, c.Len())
for i := 0; i < len(arr); i++ {
converted = append(converted, int64(arr[i]))
}
data = append(data, &schemapb.ScalarField{
Data: &schemapb.ScalarField_LongData{
LongData: &schemapb.LongArray{
Data: converted,
},
},
})
}
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_ArrayData{
ArrayData: &schemapb.ArrayArray{
Data: data,
ElementType: schemapb.DataType_Int64,
},
},
},
}
return fd
}
// ValueByIdx returns value of the provided index
// error occurs when index out of range
func (c *ColumnInt64Array) ValueByIdx(idx int) ([]int64, error) {
var r []int64 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnInt64Array) AppendValue(i interface{}) error {
v, ok := i.([]int64)
if !ok {
return fmt.Errorf("invalid type, expected []int64, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnInt64Array) Data() [][]int64 {
return c.values
}
// NewColumnInt64 auto generated constructor
func NewColumnInt64Array(name string, values [][]int64) *ColumnInt64Array {
return &ColumnInt64Array{
name: name,
values: values,
}
}
// ColumnFloatArray generated columns type for Float
type ColumnFloatArray struct {
ColumnBase
name string
values [][]float32
}
// Name returns column name
func (c *ColumnFloatArray) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnFloatArray) Type() entity.FieldType {
return entity.FieldTypeArray
}
// Len returns column values length
func (c *ColumnFloatArray) Len() int {
return len(c.values)
}
func (c *ColumnFloatArray) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &ColumnFloatArray{
ColumnBase: c.ColumnBase,
name: c.name,
values: c.values[start:end],
}
}
// Get returns value at index as interface{}.
func (c *ColumnFloatArray) Get(idx int) (interface{}, error) {
var r []float32 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnFloatArray) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_Array,
FieldName: c.name,
}
data := make([]*schemapb.ScalarField, 0, c.Len())
for _, arr := range c.values {
converted := make([]float32, 0, c.Len())
for i := 0; i < len(arr); i++ {
converted = append(converted, float32(arr[i]))
}
data = append(data, &schemapb.ScalarField{
Data: &schemapb.ScalarField_FloatData{
FloatData: &schemapb.FloatArray{
Data: converted,
},
},
})
}
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_ArrayData{
ArrayData: &schemapb.ArrayArray{
Data: data,
ElementType: schemapb.DataType_Float,
},
},
},
}
return fd
}
// ValueByIdx returns value of the provided index
// error occurs when index out of range
func (c *ColumnFloatArray) ValueByIdx(idx int) ([]float32, error) {
var r []float32 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnFloatArray) AppendValue(i interface{}) error {
v, ok := i.([]float32)
if !ok {
return fmt.Errorf("invalid type, expected []float32, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnFloatArray) Data() [][]float32 {
return c.values
}
// NewColumnFloat auto generated constructor
func NewColumnFloatArray(name string, values [][]float32) *ColumnFloatArray {
return &ColumnFloatArray{
name: name,
values: values,
}
}
// ColumnDoubleArray generated columns type for Double
type ColumnDoubleArray struct {
ColumnBase
name string
values [][]float64
}
// Name returns column name
func (c *ColumnDoubleArray) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnDoubleArray) Type() entity.FieldType {
return entity.FieldTypeArray
}
// Len returns column values length
func (c *ColumnDoubleArray) Len() int {
return len(c.values)
}
func (c *ColumnDoubleArray) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &ColumnDoubleArray{
ColumnBase: c.ColumnBase,
name: c.name,
values: c.values[start:end],
}
}
// Get returns value at index as interface{}.
func (c *ColumnDoubleArray) Get(idx int) (interface{}, error) {
var r []float64 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnDoubleArray) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_Array,
FieldName: c.name,
}
data := make([]*schemapb.ScalarField, 0, c.Len())
for _, arr := range c.values {
converted := make([]float64, 0, c.Len())
for i := 0; i < len(arr); i++ {
converted = append(converted, float64(arr[i]))
}
data = append(data, &schemapb.ScalarField{
Data: &schemapb.ScalarField_DoubleData{
DoubleData: &schemapb.DoubleArray{
Data: converted,
},
},
})
}
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_ArrayData{
ArrayData: &schemapb.ArrayArray{
Data: data,
ElementType: schemapb.DataType_Double,
},
},
},
}
return fd
}
// ValueByIdx returns value of the provided index
// error occurs when index out of range
func (c *ColumnDoubleArray) ValueByIdx(idx int) ([]float64, error) {
var r []float64 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnDoubleArray) AppendValue(i interface{}) error {
v, ok := i.([]float64)
if !ok {
return fmt.Errorf("invalid type, expected []float64, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnDoubleArray) Data() [][]float64 {
return c.values
}
// NewColumnDouble auto generated constructor
func NewColumnDoubleArray(name string, values [][]float64) *ColumnDoubleArray {
return &ColumnDoubleArray{
name: name,
values: values,
}
}

299
client/column/array_test.go Normal file
View File

@ -0,0 +1,299 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package column
import (
"fmt"
"math/rand"
"testing"
"github.com/stretchr/testify/suite"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/client/v2/entity"
)
type ArraySuite struct {
suite.Suite
}
func (s *ArraySuite) TestBasic() {
s.Run("bool_array", func() {
data := [][]bool{
{true, false},
{false, true},
}
name := fmt.Sprintf("field_%d", rand.Intn(100))
column := NewColumnBoolArray(name, data)
s.Equal(name, column.Name())
s.Equal(entity.FieldTypeArray, column.Type())
s.Equal(entity.FieldTypeBool, column.ElementType())
fd := column.FieldData()
arrayData := fd.GetScalars().GetArrayData()
s.Equal(schemapb.DataType_Bool, arrayData.GetElementType())
for i, row := range data {
sf := arrayData.GetData()[i]
s.Equal(row, sf.GetBoolData().GetData())
}
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnBoolArray)
if s.True(ok) {
s.Equal(name, parsed.Name())
s.Equal(data, parsed.Data())
s.Equal(entity.FieldTypeArray, column.Type())
s.Equal(entity.FieldTypeBool, column.ElementType())
s.Equal(data, parsed.Data())
}
})
s.Run("int8_array", func() {
data := [][]int8{
{1, 2},
{3, 4},
}
name := fmt.Sprintf("field_%d", rand.Intn(100))
column := NewColumnInt8Array(name, data)
s.Equal(name, column.Name())
s.Equal(entity.FieldTypeArray, column.Type())
s.Equal(entity.FieldTypeInt8, column.ElementType())
fd := column.FieldData()
arrayData := fd.GetScalars().GetArrayData()
s.Equal(schemapb.DataType_Int8, arrayData.GetElementType())
for i, row := range data {
sf := arrayData.GetData()[i]
for j, item := range row {
s.EqualValues(item, sf.GetIntData().GetData()[j])
}
}
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnInt8Array)
if s.True(ok) {
s.Equal(name, parsed.Name())
s.Equal(data, parsed.Data())
s.Equal(entity.FieldTypeArray, column.Type())
s.Equal(entity.FieldTypeInt8, column.ElementType())
s.Equal(data, parsed.Data())
}
})
s.Run("int16_array", func() {
data := [][]int16{
{1, 2},
{3, 4},
}
name := fmt.Sprintf("field_%d", rand.Intn(100))
column := NewColumnInt16Array(name, data)
s.Equal(name, column.Name())
s.Equal(entity.FieldTypeArray, column.Type())
s.Equal(entity.FieldTypeInt16, column.ElementType())
fd := column.FieldData()
arrayData := fd.GetScalars().GetArrayData()
s.Equal(schemapb.DataType_Int16, arrayData.GetElementType())
for i, row := range data {
sf := arrayData.GetData()[i]
for j, item := range row {
s.EqualValues(item, sf.GetIntData().GetData()[j])
}
}
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnInt16Array)
if s.True(ok) {
s.Equal(name, parsed.Name())
s.Equal(data, parsed.Data())
s.Equal(entity.FieldTypeArray, column.Type())
s.Equal(entity.FieldTypeInt16, column.ElementType())
s.Equal(data, parsed.Data())
}
})
s.Run("int32_array", func() {
data := [][]int32{
{1, 2},
{3, 4},
}
name := fmt.Sprintf("field_%d", rand.Intn(100))
column := NewColumnInt32Array(name, data)
s.Equal(name, column.Name())
s.Equal(entity.FieldTypeArray, column.Type())
s.Equal(entity.FieldTypeInt32, column.ElementType())
fd := column.FieldData()
arrayData := fd.GetScalars().GetArrayData()
s.Equal(schemapb.DataType_Int32, arrayData.GetElementType())
for i, row := range data {
sf := arrayData.GetData()[i]
s.Equal(row, sf.GetIntData().GetData())
}
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnInt32Array)
if s.True(ok) {
s.Equal(name, parsed.Name())
s.Equal(data, parsed.Data())
s.Equal(entity.FieldTypeArray, column.Type())
s.Equal(entity.FieldTypeInt32, column.ElementType())
s.Equal(data, parsed.Data())
}
})
s.Run("int64_array", func() {
data := [][]int64{
{1, 2},
{3, 4},
}
name := fmt.Sprintf("field_%d", rand.Intn(100))
column := NewColumnInt64Array(name, data)
s.Equal(name, column.Name())
s.Equal(entity.FieldTypeArray, column.Type())
s.Equal(entity.FieldTypeInt64, column.ElementType())
fd := column.FieldData()
arrayData := fd.GetScalars().GetArrayData()
s.Equal(schemapb.DataType_Int64, arrayData.GetElementType())
for i, row := range data {
sf := arrayData.GetData()[i]
s.Equal(row, sf.GetLongData().GetData())
}
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnInt64Array)
if s.True(ok) {
s.Equal(name, parsed.Name())
s.Equal(data, parsed.Data())
s.Equal(entity.FieldTypeArray, column.Type())
s.Equal(entity.FieldTypeInt64, column.ElementType())
s.Equal(data, parsed.Data())
}
})
s.Run("float_array", func() {
data := [][]float32{
{0.1, 0.2},
{1.3, 1.4},
}
name := fmt.Sprintf("field_%d", rand.Intn(100))
column := NewColumnFloatArray(name, data)
s.Equal(name, column.Name())
s.Equal(entity.FieldTypeArray, column.Type())
s.Equal(entity.FieldTypeFloat, column.ElementType())
fd := column.FieldData()
arrayData := fd.GetScalars().GetArrayData()
s.Equal(schemapb.DataType_Float, arrayData.GetElementType())
for i, row := range data {
sf := arrayData.GetData()[i]
s.Equal(row, sf.GetFloatData().GetData())
}
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnFloatArray)
if s.True(ok) {
s.Equal(name, parsed.Name())
s.Equal(data, parsed.Data())
s.Equal(entity.FieldTypeArray, column.Type())
s.Equal(entity.FieldTypeFloat, column.ElementType())
s.Equal(data, parsed.Data())
}
})
s.Run("double_array", func() {
data := [][]float64{
{0.1, 0.2},
{1.3, 1.4},
}
name := fmt.Sprintf("field_%d", rand.Intn(100))
column := NewColumnDoubleArray(name, data)
s.Equal(name, column.Name())
s.Equal(entity.FieldTypeArray, column.Type())
s.Equal(entity.FieldTypeDouble, column.ElementType())
fd := column.FieldData()
arrayData := fd.GetScalars().GetArrayData()
s.Equal(schemapb.DataType_Double, arrayData.GetElementType())
for i, row := range data {
sf := arrayData.GetData()[i]
s.Equal(row, sf.GetDoubleData().GetData())
}
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnDoubleArray)
if s.True(ok) {
s.Equal(name, parsed.Name())
s.Equal(data, parsed.Data())
s.Equal(entity.FieldTypeArray, column.Type())
s.Equal(entity.FieldTypeDouble, column.ElementType())
s.Equal(data, parsed.Data())
}
})
s.Run("varchar_array", func() {
data := [][]string{
{"abc", "def"},
{"xyz"},
}
name := fmt.Sprintf("field_%d", rand.Intn(100))
column := NewColumnVarCharArray(name, data)
s.Equal(name, column.Name())
s.Equal(entity.FieldTypeArray, column.Type())
s.Equal(entity.FieldTypeVarChar, column.ElementType())
fd := column.FieldData()
arrayData := fd.GetScalars().GetArrayData()
s.Equal(schemapb.DataType_VarChar, arrayData.GetElementType())
for i, row := range data {
sf := arrayData.GetData()[i]
s.Equal(row, sf.GetStringData().GetData())
}
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnVarCharArray)
if s.True(ok) {
s.Equal(name, parsed.Name())
s.Equal(data, parsed.Data())
s.Equal(entity.FieldTypeArray, column.Type())
s.Equal(entity.FieldTypeVarChar, column.ElementType())
s.Equal(data, parsed.Data())
}
})
}
func TestArrays(t *testing.T) {
suite.Run(t, new(ArraySuite))
}

129
client/column/columns.go
View File

@ -42,25 +42,6 @@ type Column interface {
GetAsBool(int) (bool, error)
}
// ColumnBase adds conversion methods support for fixed-type columns.
type ColumnBase struct{}
func (b ColumnBase) GetAsInt64(_ int) (int64, error) {
return 0, errors.New("conversion between fixed-type column not support")
}
func (b ColumnBase) GetAsString(_ int) (string, error) {
return "", errors.New("conversion between fixed-type column not support")
}
func (b ColumnBase) GetAsDouble(_ int) (float64, error) {
return 0, errors.New("conversion between fixed-type column not support")
}
func (b ColumnBase) GetAsBool(_ int) (bool, error) {
return false, errors.New("conversion between fixed-type column not support")
}
var errFieldDataTypeNotMatch = errors.New("FieldData type not matched")
// IDColumns converts schemapb.IDs to corresponding column
@ -393,13 +374,12 @@ func parseArrayData(fieldName string, elementType schemapb.DataType, fieldDataLi
return NewColumnDoubleArray(fieldName, data), nil
case schemapb.DataType_VarChar, schemapb.DataType_String:
var data [][][]byte
var data [][]string
for _, fd := range fieldDataList {
strs := fd.GetStringData().GetData()
bytesData := make([][]byte, 0, len(strs))
for _, str := range strs {
bytesData = append(bytesData, []byte(str))
}
bytesData := make([]string, 0, len(strs))
bytesData = append(bytesData, strs...)
data = append(data, bytesData)
}
@ -428,104 +408,3 @@ func getIntData(fd *schemapb.FieldData) (*schemapb.ScalarField_IntData, bool) {
return nil, false
}
}
// FieldDataColumn converts schemapb.FieldData to vector Column
func FieldDataVector(fd *schemapb.FieldData) (Column, error) {
switch fd.GetType() {
case schemapb.DataType_FloatVector:
vectors := fd.GetVectors()
x, ok := vectors.GetData().(*schemapb.VectorField_FloatVector)
if !ok {
return nil, errFieldDataTypeNotMatch
}
data := x.FloatVector.GetData()
dim := int(vectors.GetDim())
vector := make([][]float32, 0, len(data)/dim) // shall not have remanunt
for i := 0; i < len(data)/dim; i++ {
v := make([]float32, dim)
copy(v, data[i*dim:(i+1)*dim])
vector = append(vector, v)
}
return NewColumnFloatVector(fd.GetFieldName(), dim, vector), nil
case schemapb.DataType_BinaryVector:
vectors := fd.GetVectors()
x, ok := vectors.GetData().(*schemapb.VectorField_BinaryVector)
if !ok {
return nil, errFieldDataTypeNotMatch
}
data := x.BinaryVector
if data == nil {
return nil, errFieldDataTypeNotMatch
}
dim := int(vectors.GetDim())
blen := dim / 8
vector := make([][]byte, 0, len(data)/blen)
for i := 0; i < len(data)/blen; i++ {
v := make([]byte, blen)
copy(v, data[i*blen:(i+1)*blen])
vector = append(vector, v)
}
return NewColumnBinaryVector(fd.GetFieldName(), dim, vector), nil
case schemapb.DataType_Float16Vector:
vectors := fd.GetVectors()
x, ok := vectors.GetData().(*schemapb.VectorField_Float16Vector)
if !ok {
return nil, errFieldDataTypeNotMatch
}
data := x.Float16Vector
dim := int(vectors.GetDim())
vector := make([][]byte, 0, len(data)/dim) // shall not have remanunt
for i := 0; i < len(data)/dim; i++ {
v := make([]byte, dim)
copy(v, data[i*dim:(i+1)*dim])
vector = append(vector, v)
}
return NewColumnFloat16Vector(fd.GetFieldName(), dim, vector), nil
case schemapb.DataType_BFloat16Vector:
vectors := fd.GetVectors()
x, ok := vectors.GetData().(*schemapb.VectorField_Bfloat16Vector)
if !ok {
return nil, errFieldDataTypeNotMatch
}
data := x.Bfloat16Vector
dim := int(vectors.GetDim())
vector := make([][]byte, 0, len(data)/dim) // shall not have remanunt
for i := 0; i < len(data)/dim; i++ {
v := make([]byte, dim)
copy(v, data[i*dim:(i+1)*dim])
vector = append(vector, v)
}
return NewColumnBFloat16Vector(fd.GetFieldName(), dim, vector), nil
default:
return nil, errors.New("unsupported data type")
}
}
// defaultValueColumn will return the empty scalars column which will be fill with default value
func DefaultValueColumn(name string, dataType entity.FieldType) (Column, error) {
switch dataType {
case entity.FieldTypeBool:
return NewColumnBool(name, nil), nil
case entity.FieldTypeInt8:
return NewColumnInt8(name, nil), nil
case entity.FieldTypeInt16:
return NewColumnInt16(name, nil), nil
case entity.FieldTypeInt32:
return NewColumnInt32(name, nil), nil
case entity.FieldTypeInt64:
return NewColumnInt64(name, nil), nil
case entity.FieldTypeFloat:
return NewColumnFloat(name, nil), nil
case entity.FieldTypeDouble:
return NewColumnDouble(name, nil), nil
case entity.FieldTypeString:
return NewColumnString(name, nil), nil
case entity.FieldTypeVarChar:
return NewColumnVarChar(name, nil), nil
case entity.FieldTypeJSON:
return NewColumnJSONBytes(name, nil), nil
default:
return nil, fmt.Errorf("default value unsupported data type %s", dataType)
}
}

283
client/column/conversion.go
View File

@ -16,38 +16,269 @@
package column
func (c *ColumnInt8) GetAsInt64(idx int) (int64, error) {
v, err := c.ValueByIdx(idx)
return int64(v), err
import (
"fmt"
"github.com/cockroachdb/errors"
"github.com/samber/lo"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/client/v2/entity"
)
func slice2Scalar[T any](values []T, elementType entity.FieldType) *schemapb.ScalarField {
var ok bool
scalarField := &schemapb.ScalarField{}
switch elementType {
case entity.FieldTypeBool:
var bools []bool
bools, ok = any(values).([]bool)
scalarField.Data = &schemapb.ScalarField_BoolData{
BoolData: &schemapb.BoolArray{
Data: bools,
},
}
case entity.FieldTypeInt8:
var int8s []int8
int8s, ok = any(values).([]int8)
int32s := lo.Map(int8s, func(i8 int8, _ int) int32 { return int32(i8) })
scalarField.Data = &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: int32s,
},
}
case entity.FieldTypeInt16:
var int16s []int16
int16s, ok = any(values).([]int16)
int32s := lo.Map(int16s, func(i16 int16, _ int) int32 { return int32(i16) })
scalarField.Data = &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: int32s,
},
}
case entity.FieldTypeInt32:
var int32s []int32
int32s, ok = any(values).([]int32)
scalarField.Data = &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: int32s,
},
}
case entity.FieldTypeInt64:
var int64s []int64
int64s, ok = any(values).([]int64)
scalarField.Data = &schemapb.ScalarField_LongData{
LongData: &schemapb.LongArray{
Data: int64s,
},
}
case entity.FieldTypeFloat:
var floats []float32
floats, ok = any(values).([]float32)
scalarField.Data = &schemapb.ScalarField_FloatData{
FloatData: &schemapb.FloatArray{
Data: floats,
},
}
case entity.FieldTypeDouble:
var doubles []float64
doubles, ok = any(values).([]float64)
scalarField.Data = &schemapb.ScalarField_DoubleData{
DoubleData: &schemapb.DoubleArray{
Data: doubles,
},
}
case entity.FieldTypeVarChar, entity.FieldTypeString:
var strings []string
strings, ok = any(values).([]string)
scalarField.Data = &schemapb.ScalarField_StringData{
StringData: &schemapb.StringArray{
Data: strings,
},
}
}
if !ok {
panic(fmt.Sprintf("unexpected values type(%T) of fieldType %v", values, elementType))
}
return scalarField
}
func (c *ColumnInt16) GetAsInt64(idx int) (int64, error) {
v, err := c.ValueByIdx(idx)
return int64(v), err
func values2FieldData[T any](values []T, fieldType entity.FieldType, dim int) *schemapb.FieldData {
fd := &schemapb.FieldData{}
switch fieldType {
// scalars
case entity.FieldTypeBool,
entity.FieldTypeFloat,
entity.FieldTypeDouble,
entity.FieldTypeInt8,
entity.FieldTypeInt16,
entity.FieldTypeInt32,
entity.FieldTypeInt64,
entity.FieldTypeVarChar,
entity.FieldTypeString,
entity.FieldTypeJSON:
fd.Field = &schemapb.FieldData_Scalars{
Scalars: values2Scalars(values, fieldType), // scalars,
}
// vectors
case entity.FieldTypeFloatVector,
entity.FieldTypeFloat16Vector,
entity.FieldTypeBFloat16Vector,
entity.FieldTypeBinaryVector,
entity.FieldTypeSparseVector:
fd.Field = &schemapb.FieldData_Vectors{
Vectors: values2Vectors(values, fieldType, int64(dim)),
}
default:
panic(fmt.Sprintf("unexpected values type(%T) of fieldType %v", values, fieldType))
}
return fd
}
func (c *ColumnInt32) GetAsInt64(idx int) (int64, error) {
v, err := c.ValueByIdx(idx)
return int64(v), err
func values2Scalars[T any](values []T, fieldType entity.FieldType) *schemapb.ScalarField {
scalars := &schemapb.ScalarField{}
var ok bool
switch fieldType {
case entity.FieldTypeBool:
var bools []bool
bools, ok = any(values).([]bool)
scalars.Data = &schemapb.ScalarField_BoolData{
BoolData: &schemapb.BoolArray{Data: bools},
}
case entity.FieldTypeInt8:
var int8s []int8
int8s, ok = any(values).([]int8)
int32s := lo.Map(int8s, func(i8 int8, _ int) int32 { return int32(i8) })
scalars.Data = &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{Data: int32s},
}
case entity.FieldTypeInt16:
var int16s []int16
int16s, ok = any(values).([]int16)
int32s := lo.Map(int16s, func(i16 int16, _ int) int32 { return int32(i16) })
scalars.Data = &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{Data: int32s},
}
case entity.FieldTypeInt32:
var int32s []int32
int32s, ok = any(values).([]int32)
scalars.Data = &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{Data: int32s},
}
case entity.FieldTypeInt64:
var int64s []int64
int64s, ok = any(values).([]int64)
scalars.Data = &schemapb.ScalarField_LongData{
LongData: &schemapb.LongArray{Data: int64s},
}
case entity.FieldTypeVarChar, entity.FieldTypeString:
var strVals []string
strVals, ok = any(values).([]string)
scalars.Data = &schemapb.ScalarField_StringData{
StringData: &schemapb.StringArray{Data: strVals},
}
case entity.FieldTypeFloat:
var floats []float32
floats, ok = any(values).([]float32)
scalars.Data = &schemapb.ScalarField_FloatData{
FloatData: &schemapb.FloatArray{Data: floats},
}
case entity.FieldTypeDouble:
var data []float64
data, ok = any(values).([]float64)
scalars.Data = &schemapb.ScalarField_DoubleData{
DoubleData: &schemapb.DoubleArray{Data: data},
}
case entity.FieldTypeJSON:
var data [][]byte
data, ok = any(values).([][]byte)
scalars.Data = &schemapb.ScalarField_JsonData{
JsonData: &schemapb.JSONArray{
Data: data,
},
}
}
// shall not be accessed
if !ok {
panic(fmt.Sprintf("unexpected values type(%T) of fieldType %v", values, fieldType))
}
return scalars
}
func (c *ColumnInt64) GetAsInt64(idx int) (int64, error) {
return c.ValueByIdx(idx)
func values2Vectors[T any](values []T, fieldType entity.FieldType, dim int64) *schemapb.VectorField {
vectorField := &schemapb.VectorField{
Dim: dim,
}
var ok bool
switch fieldType {
case entity.FieldTypeFloatVector:
var vectors []entity.FloatVector
vectors, ok = any(values).([]entity.FloatVector)
data := make([]float32, 0, int64(len(vectors))*dim)
for _, vector := range vectors {
data = append(data, vector...)
}
vectorField.Data = &schemapb.VectorField_FloatVector{
FloatVector: &schemapb.FloatArray{
Data: data,
},
}
case entity.FieldTypeFloat16Vector:
var vectors []entity.Float16Vector
vectors, ok = any(values).([]entity.Float16Vector)
data := make([]byte, 0, int64(len(vectors))*dim*2)
for _, vector := range vectors {
data = append(data, vector.Serialize()...)
}
vectorField.Data = &schemapb.VectorField_Float16Vector{
Float16Vector: data,
}
case entity.FieldTypeBFloat16Vector:
var vectors []entity.BFloat16Vector
vectors, ok = any(values).([]entity.BFloat16Vector)
data := make([]byte, 0, int64(len(vectors))*dim*2)
for _, vector := range vectors {
data = append(data, vector.Serialize()...)
}
vectorField.Data = &schemapb.VectorField_Bfloat16Vector{
Bfloat16Vector: data,
}
case entity.FieldTypeBinaryVector:
var vectors []entity.BinaryVector
vectors, ok = any(values).([]entity.BinaryVector)
data := make([]byte, 0, int64(len(vectors))*dim/8)
for _, vector := range vectors {
data = append(data, vector.Serialize()...)
}
vectorField.Data = &schemapb.VectorField_BinaryVector{
BinaryVector: data,
}
case entity.FieldTypeSparseVector:
var vectors []entity.SparseEmbedding
vectors, ok = any(values).([]entity.SparseEmbedding)
data := lo.Map(vectors, func(row entity.SparseEmbedding, _ int) []byte {
return row.Serialize()
})
vectorField.Data = &schemapb.VectorField_SparseFloatVector{
SparseFloatVector: &schemapb.SparseFloatArray{
Contents: data,
},
}
}
if !ok {
panic(fmt.Sprintf("unexpected values type(%T) of fieldType %v", values, fieldType))
}
return vectorField
}
func (c *ColumnString) GetAsString(idx int) (string, error) {
return c.ValueByIdx(idx)
}
func (c *ColumnFloat) GetAsDouble(idx int) (float64, error) {
v, err := c.ValueByIdx(idx)
return float64(v), err
}
func (c *ColumnDouble) GetAsDouble(idx int) (float64, error) {
return c.ValueByIdx(idx)
}
func (c *ColumnBool) GetAsBool(idx int) (bool, error) {
return c.ValueByIdx(idx)
func value2Type[T any, U any](v T) (U, error) {
var z U
switch v := any(v).(type) {
case U:
return v, nil
default:
return z, errors.Newf("cannot automatically convert %T to %T", v, z)
}
}

10
client/column/dynamic.go
View File

@ -41,7 +41,7 @@ func (c *ColumnDynamic) Name() string {
// Get returns element at idx as interface{}.
// Overrides internal json column behavior, returns raw json data.
func (c *ColumnDynamic) Get(idx int) (interface{}, error) {
bs, err := c.ColumnJSONBytes.ValueByIdx(idx)
bs, err := c.ColumnJSONBytes.Value(idx)
if err != nil {
return 0, err
}
@ -53,7 +53,7 @@ func (c *ColumnDynamic) Get(idx int) (interface{}, error) {
}
func (c *ColumnDynamic) GetAsInt64(idx int) (int64, error) {
bs, err := c.ColumnJSONBytes.ValueByIdx(idx)
bs, err := c.ColumnJSONBytes.Value(idx)
if err != nil {
return 0, err
}
@ -68,7 +68,7 @@ func (c *ColumnDynamic) GetAsInt64(idx int) (int64, error) {
}
func (c *ColumnDynamic) GetAsString(idx int) (string, error) {
bs, err := c.ColumnJSONBytes.ValueByIdx(idx)
bs, err := c.ColumnJSONBytes.Value(idx)
if err != nil {
return "", err
}
@ -83,7 +83,7 @@ func (c *ColumnDynamic) GetAsString(idx int) (string, error) {
}
func (c *ColumnDynamic) GetAsBool(idx int) (bool, error) {
bs, err := c.ColumnJSONBytes.ValueByIdx(idx)
bs, err := c.ColumnJSONBytes.Value(idx)
if err != nil {
return false, err
}
@ -98,7 +98,7 @@ func (c *ColumnDynamic) GetAsBool(idx int) (bool, error) {
}
func (c *ColumnDynamic) GetAsDouble(idx int) (float64, error) {
bs, err := c.ColumnJSONBytes.ValueByIdx(idx)
bs, err := c.ColumnJSONBytes.Value(idx)
if err != nil {
return 0, err
}

27
client/column/dynamic_test.go
View File

@ -40,9 +40,7 @@ func (s *ColumnDynamicSuite) TestGetInt() {
for _, c := range cases {
s.Run(c.input, func() {
column := NewColumnDynamic(&ColumnJSONBytes{
values: [][]byte{[]byte(c.input)},
}, "field")
column := NewColumnDynamic(NewColumnJSONBytes("", [][]byte{[]byte(c.input)}), "field")
v, err := column.GetAsInt64(0)
if c.expectErr {
s.Error(err)
@ -68,9 +66,10 @@ func (s *ColumnDynamicSuite) TestGetString() {
for _, c := range cases {
s.Run(c.input, func() {
column := NewColumnDynamic(&ColumnJSONBytes{
values: [][]byte{[]byte(c.input)},
}, "field")
// column := NewColumnDynamic(&ColumnJSONBytes{
// values: [][]byte{[]byte(c.input)},
// }, "field")
column := NewColumnDynamic(NewColumnJSONBytes("", [][]byte{[]byte(c.input)}), "field")
v, err := column.GetAsString(0)
if c.expectErr {
s.Error(err)
@ -96,9 +95,10 @@ func (s *ColumnDynamicSuite) TestGetBool() {
for _, c := range cases {
s.Run(c.input, func() {
column := NewColumnDynamic(&ColumnJSONBytes{
values: [][]byte{[]byte(c.input)},
}, "field")
// column := NewColumnDynamic(&ColumnJSONBytes{
// values: [][]byte{[]byte(c.input)},
// }, "field")
column := NewColumnDynamic(NewColumnJSONBytes("", [][]byte{[]byte(c.input)}), "field")
v, err := column.GetAsBool(0)
if c.expectErr {
s.Error(err)
@ -124,9 +124,10 @@ func (s *ColumnDynamicSuite) TestGetDouble() {
for _, c := range cases {
s.Run(c.input, func() {
column := NewColumnDynamic(&ColumnJSONBytes{
values: [][]byte{[]byte(c.input)},
}, "field")
// column := NewColumnDynamic(&ColumnJSONBytes{
// values: [][]byte{[]byte(c.input)},
// }, "field")
column := NewColumnDynamic(NewColumnJSONBytes("", [][]byte{[]byte(c.input)}), "field")
v, err := column.GetAsDouble(0)
if c.expectErr {
s.Error(err)
@ -140,7 +141,7 @@ func (s *ColumnDynamicSuite) TestGetDouble() {
func (s *ColumnDynamicSuite) TestIndexOutOfRange() {
var err error
column := NewColumnDynamic(&ColumnJSONBytes{}, "field")
column := NewColumnDynamic(NewColumnJSONBytes("", nil), "field")
s.Equal("field", column.Name())

146
client/column/generic_base.go Normal file
View File

@ -0,0 +1,146 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package column
import (
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/client/v2/entity"
)
var _ Column = (*genericColumnBase[any])(nil)
// genericColumnBase implements `Column` interface
// it provides the basic function for each scalar params
type genericColumnBase[T any] struct {
name string
fieldType entity.FieldType
values []T
}
// Name returns column name.
func (c *genericColumnBase[T]) Name() string {
return c.name
}
// Type returns corresponding field type.
// note that: it is not necessary to be 1-on-1 mapping
// say, `[]byte` could be lots of field type.
func (c *genericColumnBase[T]) Type() entity.FieldType {
return c.fieldType
}
func (c *genericColumnBase[T]) Len() int {
return len(c.values)
}
func (c *genericColumnBase[T]) AppendValue(a any) error {
v, ok := a.(T)
if !ok {
return errors.Newf("unexpected append value type %T, field type %v", a, c.fieldType)
}
c.values = append(c.values, v)
return nil
}
func (c *genericColumnBase[T]) Slice(start, end int) Column {
return c.slice(start, end)
}
func (c *genericColumnBase[T]) slice(start, end int) *genericColumnBase[T] {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &genericColumnBase[T]{
name: c.name,
fieldType: c.fieldType,
values: c.values[start:end],
}
}
func (c *genericColumnBase[T]) FieldData() *schemapb.FieldData {
fd := values2FieldData(c.values, c.fieldType, 0)
fd.FieldName = c.name
fd.Type = schemapb.DataType(c.fieldType)
return fd
}
func (c *genericColumnBase[T]) rangeCheck(idx int) error {
if idx < 0 || idx >= c.Len() {
return errors.Newf("index %d out of range[0, %d)", idx, c.Len())
}
return nil
}
func (c *genericColumnBase[T]) Get(idx int) (any, error) {
if err := c.rangeCheck(idx); err != nil {
return nil, err
}
return c.values[idx], nil
}
func (c *genericColumnBase[T]) GetAsInt64(idx int) (int64, error) {
if err := c.rangeCheck(idx); err != nil {
return 0, err
}
return value2Type[T, int64](c.values[idx])
}
func (c *genericColumnBase[T]) GetAsString(idx int) (string, error) {
if err := c.rangeCheck(idx); err != nil {
return "", err
}
return value2Type[T, string](c.values[idx])
}
func (c *genericColumnBase[T]) GetAsDouble(idx int) (float64, error) {
if err := c.rangeCheck(idx); err != nil {
return 0, err
}
return value2Type[T, float64](c.values[idx])
}
func (c *genericColumnBase[T]) GetAsBool(idx int) (bool, error) {
if err := c.rangeCheck(idx); err != nil {
return false, err
}
return value2Type[T, bool](c.values[idx])
}
func (c *genericColumnBase[T]) Value(idx int) (T, error) {
var z T
if err := c.rangeCheck(idx); err != nil {
return z, err
}
return c.values[idx], nil
}
func (c *genericColumnBase[T]) Data() []T {
return c.values
}
func (c *genericColumnBase[T]) MustValue(idx int) T {
if idx < 0 || idx > c.Len() {
panic("index out of range")
}
return c.values[idx]
}

153
client/column/generic_base_test.go Normal file
View File

@ -0,0 +1,153 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package column
import (
"fmt"
"math/rand"
"testing"
"github.com/stretchr/testify/suite"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/client/v2/entity"
)
type GenericBaseSuite struct {
suite.Suite
}
func (s *GenericBaseSuite) TestBasic() {
name := fmt.Sprintf("test_%d", rand.Intn(10))
gb := &genericColumnBase[int64]{
name: name,
fieldType: entity.FieldTypeInt64,
values: []int64{1, 2, 3},
}
s.Equal(name, gb.Name())
s.Equal(entity.FieldTypeInt64, gb.Type())
s.EqualValues(3, gb.Len())
err := gb.AppendValue("abc")
s.Error(err)
s.EqualValues(3, gb.Len())
err = gb.AppendValue(int64(4))
s.NoError(err)
s.EqualValues(4, gb.Len())
}
func (s *GenericBaseSuite) TestIndexAccess() {
name := fmt.Sprintf("test_%d", rand.Intn(10))
values := []int64{1, 2, 3}
gb := &genericColumnBase[int64]{
name: name,
fieldType: entity.FieldTypeInt64,
values: values,
}
for idx, value := range values {
v, err := gb.Value(idx)
s.NoError(err)
s.Equal(value, v)
s.NotPanics(func() {
v = gb.MustValue(idx)
})
s.Equal(value, v)
}
s.Panics(func() {
gb.MustValue(-1)
}, "out of range, negative index")
s.Panics(func() {
gb.MustValue(3)
}, "out of range, LTE len")
s.NotPanics(func() {
_, err := gb.Value(-1)
s.Error(err)
_, err = gb.Value(3)
s.Error(err)
})
}
func (s *GenericBaseSuite) TestSlice() {
name := fmt.Sprintf("test_%d", rand.Intn(10))
values := []int64{1, 2, 3}
gb := &genericColumnBase[int64]{
name: name,
fieldType: entity.FieldTypeInt64,
values: values,
}
l := rand.Intn(3)
another := gb.Slice(0, l)
s.Equal(l, another.Len())
agb, ok := another.(*genericColumnBase[int64])
s.Require().True(ok)
for i := 0; i < l; i++ {
s.Equal(gb.MustValue(i), agb.MustValue(i))
}
s.NotPanics(func() {
agb := gb.Slice(10, 10)
s.Equal(0, agb.Len())
})
}
func (s *GenericBaseSuite) TestFieldData() {
name := fmt.Sprintf("test_%d", rand.Intn(10))
values := []int64{1, 2, 3}
gb := &genericColumnBase[int64]{
name: name,
fieldType: entity.FieldTypeInt64,
values: values,
}
fd := gb.FieldData()
s.Equal(name, fd.GetFieldName())
s.Equal(schemapb.DataType_Int64, fd.GetType())
}
func (s *GenericBaseSuite) TestConversion() {
name := fmt.Sprintf("test_%d", rand.Intn(10))
values := []int64{1, 2, 3}
gb := &genericColumnBase[int64]{
name: name,
fieldType: entity.FieldTypeInt64,
values: values,
}
val, err := gb.GetAsInt64(0)
s.NoError(err)
s.EqualValues(1, val)
_, err = gb.GetAsBool(0)
s.Error(err)
_, err = gb.GetAsBool(0)
s.Error(err)
}
func TestGenericBase(t *testing.T) {
suite.Run(t, new(GenericBaseSuite))
}

104
client/column/json.go
View File

@ -21,98 +21,44 @@ import (
"fmt"
"reflect"
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/client/v2/entity"
)
var _ (Column) = (*ColumnJSONBytes)(nil)
var _ (Column) = (*ColumnVarChar)(nil)
// ColumnJSONBytes column type for JSON.
// all items are marshaled json bytes.
type ColumnJSONBytes struct {
ColumnBase
name string
values [][]byte
*genericColumnBase[[]byte]
isDynamic bool
}
// Name returns column name.
func (c *ColumnJSONBytes) Name() string {
return c.name
}
// Type returns column entity.FieldType.
func (c *ColumnJSONBytes) Type() entity.FieldType {
return entity.FieldTypeJSON
}
// Len returns column values length.
func (c *ColumnJSONBytes) Len() int {
return len(c.values)
func NewColumnJSONBytes(name string, values [][]byte) *ColumnJSONBytes {
return &ColumnJSONBytes{
genericColumnBase: &genericColumnBase[[]byte]{
name: name,
fieldType: entity.FieldTypeJSON,
values: values,
},
}
}
func (c *ColumnJSONBytes) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &ColumnJSONBytes{
ColumnBase: c.ColumnBase,
name: c.name,
values: c.values[start:end],
genericColumnBase: c.genericColumnBase.slice(start, end),
}
}
// Get returns value at index as interface{}.
func (c *ColumnJSONBytes) Get(idx int) (interface{}, error) {
if idx < 0 || idx > c.Len() {
return nil, errors.New("index out of range")
}
return c.values[idx], nil
func (c *ColumnJSONBytes) WithIsDynamic(isDynamic bool) *ColumnJSONBytes {
c.isDynamic = isDynamic
return c
}
func (c *ColumnJSONBytes) GetAsString(idx int) (string, error) {
bs, err := c.ValueByIdx(idx)
if err != nil {
return "", err
}
return string(bs), nil
}
// FieldData return column data mapped to schemapb.FieldData.
func (c *ColumnJSONBytes) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_JSON,
FieldName: c.name,
IsDynamic: c.isDynamic,
}
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_JsonData{
JsonData: &schemapb.JSONArray{
Data: c.values,
},
},
},
}
fd := c.genericColumnBase.FieldData()
fd.IsDynamic = c.isDynamic
return fd
}
// ValueByIdx returns value of the provided index.
func (c *ColumnJSONBytes) ValueByIdx(idx int) ([]byte, error) {
if idx < 0 || idx >= c.Len() {
return nil, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column.
func (c *ColumnJSONBytes) AppendValue(i interface{}) error {
var v []byte
@ -141,21 +87,3 @@ func (c *ColumnJSONBytes) AppendValue(i interface{}) error {
return nil
}
// Data returns column data.
func (c *ColumnJSONBytes) Data() [][]byte {
return c.values
}
func (c *ColumnJSONBytes) WithIsDynamic(isDynamic bool) *ColumnJSONBytes {
c.isDynamic = isDynamic
return c
}
// NewColumnJSONBytes composes a Column with json bytes.
func NewColumnJSONBytes(name string, values [][]byte) *ColumnJSONBytes {
return &ColumnJSONBytes{
name: name,
values: values,
}
}

17
client/column/json_test.go
View File

@ -62,15 +62,24 @@ func (s *ColumnJSONBytesSuite) TestAttrMethods() {
fd := column.FieldData()
s.NotNil(fd)
s.Equal(fd.GetFieldName(), columnName)
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnJSONBytes)
if s.True(ok) {
s.Equal(columnName, parsed.Name())
s.Equal(entity.FieldTypeJSON, parsed.Type())
s.Equal(v, parsed.Data())
}
})
s.Run("test_column_valuer_by_idx", func() {
_, err := column.ValueByIdx(-1)
_, err := column.Value(-1)
s.Error(err)
_, err = column.ValueByIdx(columnLen)
_, err = column.Value(columnLen)
s.Error(err)
for i := 0; i < columnLen; i++ {
v, err := column.ValueByIdx(i)
v, err := column.Value(i)
s.NoError(err)
s.Equal(column.values[i], v)
}
@ -81,7 +90,7 @@ func (s *ColumnJSONBytesSuite) TestAttrMethods() {
err := column.AppendValue(item)
s.NoError(err)
s.Equal(columnLen+1, column.Len())
val, err := column.ValueByIdx(columnLen)
val, err := column.Value(columnLen)
s.NoError(err)
s.Equal(item, val)

256
client/column/scalar.go Normal file
View File

@ -0,0 +1,256 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package column
import "github.com/milvus-io/milvus/client/v2/entity"
/* bool */
var _ Column = (*ColumnBool)(nil)
type ColumnBool struct {
*genericColumnBase[bool]
}
func NewColumnBool(name string, values []bool) *ColumnBool {
return &ColumnBool{
genericColumnBase: &genericColumnBase[bool]{
name: name,
fieldType: entity.FieldTypeBool,
values: values,
},
}
}
func (c *ColumnBool) Slice(start, end int) Column {
return &ColumnBool{
genericColumnBase: c.genericColumnBase.slice(start, end),
}
}
/* Int8 */
var _ Column = (*ColumnInt8)(nil)
type ColumnInt8 struct {
*genericColumnBase[int8]
}
func NewColumnInt8(name string, values []int8) *ColumnInt8 {
return &ColumnInt8{
genericColumnBase: &genericColumnBase[int8]{
name: name,
fieldType: entity.FieldTypeInt8,
values: values,
},
}
}
func (c *ColumnInt8) Slice(start, end int) Column {
return &ColumnInt8{
genericColumnBase: c.genericColumnBase.slice(start, end),
}
}
func (c *ColumnInt8) GetAsInt64(idx int) (int64, error) {
v, err := c.Value(idx)
return int64(v), err
}
/* Int16 */
var _ Column = (*ColumnInt16)(nil)
type ColumnInt16 struct {
*genericColumnBase[int16]
}
func NewColumnInt16(name string, values []int16) *ColumnInt16 {
return &ColumnInt16{
genericColumnBase: &genericColumnBase[int16]{
name: name,
fieldType: entity.FieldTypeInt16,
values: values,
},
}
}
func (c *ColumnInt16) Slice(start, end int) Column {
return &ColumnInt16{
genericColumnBase: c.genericColumnBase.slice(start, end),
}
}
func (c *ColumnInt16) GetAsInt64(idx int) (int64, error) {
v, err := c.Value(idx)
return int64(v), err
}
/* Int32 */
var _ Column = (*ColumnInt32)(nil)
type ColumnInt32 struct {
*genericColumnBase[int32]
}
func NewColumnInt32(name string, values []int32) *ColumnInt32 {
return &ColumnInt32{
genericColumnBase: &genericColumnBase[int32]{
name: name,
fieldType: entity.FieldTypeInt32,
values: values,
},
}
}
func (c *ColumnInt32) Slice(start, end int) Column {
return &ColumnInt32{
genericColumnBase: c.genericColumnBase.slice(start, end),
}
}
func (c *ColumnInt32) GetAsInt64(idx int) (int64, error) {
v, err := c.Value(idx)
return int64(v), err
}
/* Int64 */
var _ Column = (*ColumnInt64)(nil)
type ColumnInt64 struct {
*genericColumnBase[int64]
}
func NewColumnInt64(name string, values []int64) *ColumnInt64 {
return &ColumnInt64{
genericColumnBase: &genericColumnBase[int64]{
name: name,
fieldType: entity.FieldTypeInt64,
values: values,
},
}
}
func (c *ColumnInt64) Slice(start, end int) Column {
return &ColumnInt64{
genericColumnBase: c.genericColumnBase.slice(start, end),
}
}
/* Float */
var _ Column = (*ColumnFloat)(nil)
type ColumnFloat struct {
*genericColumnBase[float32]
}
func NewColumnFloat(name string, values []float32) *ColumnFloat {
return &ColumnFloat{
genericColumnBase: &genericColumnBase[float32]{
name: name,
fieldType: entity.FieldTypeFloat,
values: values,
},
}
}
func (c *ColumnFloat) Slice(start, end int) Column {
return &ColumnFloat{
genericColumnBase: c.genericColumnBase.slice(start, end),
}
}
func (c *ColumnFloat) GetAsDouble(idx int) (float64, error) {
v, err := c.Value(idx)
return float64(v), err
}
/* Double */
var _ Column = (*ColumnFloat)(nil)
type ColumnDouble struct {
*genericColumnBase[float64]
}
func NewColumnDouble(name string, values []float64) *ColumnDouble {
return &ColumnDouble{
genericColumnBase: &genericColumnBase[float64]{
name: name,
fieldType: entity.FieldTypeDouble,
values: values,
},
}
}
func (c *ColumnDouble) Slice(start, end int) Column {
return &ColumnDouble{
genericColumnBase: c.genericColumnBase.slice(start, end),
}
}
/* Varchar */
var _ (Column) = (*ColumnVarChar)(nil)
type ColumnVarChar struct {
*genericColumnBase[string]
}
func NewColumnVarChar(name string, values []string) *ColumnVarChar {
return &ColumnVarChar{
genericColumnBase: &genericColumnBase[string]{
name: name,
fieldType: entity.FieldTypeVarChar,
values: values,
},
}
}
func (c *ColumnVarChar) Slice(start, end int) Column {
return &ColumnVarChar{
genericColumnBase: c.genericColumnBase.slice(start, end),
}
}
/* String */
/* NOT USED */
var _ (Column) = (*ColumnString)(nil)
type ColumnString struct {
*genericColumnBase[string]
}
func NewColumnString(name string, values []string) *ColumnString {
return &ColumnString{
genericColumnBase: &genericColumnBase[string]{
name: name,
fieldType: entity.FieldTypeString,
values: values,
},
}
}
func (c *ColumnString) Slice(start, end int) Column {
return &ColumnString{
genericColumnBase: c.genericColumnBase.slice(start, end),
}
}

828
client/column/scalar_gen.go
View File

@ -1,828 +0,0 @@
// Code generated by go generate; DO NOT EDIT
// This file is generated by go generate
package column
import (
"errors"
"fmt"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/client/v2/entity"
)
// ColumnBool generated columns type for Bool
type ColumnBool struct {
ColumnBase
name string
values []bool
}
// Name returns column name
func (c *ColumnBool) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnBool) Type() entity.FieldType {
return entity.FieldTypeBool
}
// Len returns column values length
func (c *ColumnBool) Len() int {
return len(c.values)
}
func (c *ColumnBool) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &ColumnBool{
ColumnBase: c.ColumnBase,
name: c.name,
values: c.values[start:end],
}
}
// Get returns value at index as interface{}.
func (c *ColumnBool) Get(idx int) (interface{}, error) {
var r bool // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnBool) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_Bool,
FieldName: c.name,
}
data := make([]bool, 0, c.Len())
for i := 0; i < c.Len(); i++ {
data = append(data, bool(c.values[i]))
}
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_BoolData{
BoolData: &schemapb.BoolArray{
Data: data,
},
},
},
}
return fd
}
// ValueByIdx returns value of the provided index
// error occurs when index out of range
func (c *ColumnBool) ValueByIdx(idx int) (bool, error) {
var r bool // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnBool) AppendValue(i interface{}) error {
v, ok := i.(bool)
if !ok {
return fmt.Errorf("invalid type, expected bool, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnBool) Data() []bool {
return c.values
}
// NewColumnBool auto generated constructor
func NewColumnBool(name string, values []bool) *ColumnBool {
return &ColumnBool{
name: name,
values: values,
}
}
// ColumnInt8 generated columns type for Int8
type ColumnInt8 struct {
ColumnBase
name string
values []int8
}
// Name returns column name
func (c *ColumnInt8) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnInt8) Type() entity.FieldType {
return entity.FieldTypeInt8
}
// Len returns column values length
func (c *ColumnInt8) Len() int {
return len(c.values)
}
func (c *ColumnInt8) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &ColumnInt8{
ColumnBase: c.ColumnBase,
name: c.name,
values: c.values[start:end],
}
}
// Get returns value at index as interface{}.
func (c *ColumnInt8) Get(idx int) (interface{}, error) {
var r int8 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnInt8) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_Int8,
FieldName: c.name,
}
data := make([]int32, 0, c.Len())
for i := 0; i < c.Len(); i++ {
data = append(data, int32(c.values[i]))
}
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: data,
},
},
},
}
return fd
}
// ValueByIdx returns value of the provided index
// error occurs when index out of range
func (c *ColumnInt8) ValueByIdx(idx int) (int8, error) {
var r int8 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnInt8) AppendValue(i interface{}) error {
v, ok := i.(int8)
if !ok {
return fmt.Errorf("invalid type, expected int8, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnInt8) Data() []int8 {
return c.values
}
// NewColumnInt8 auto generated constructor
func NewColumnInt8(name string, values []int8) *ColumnInt8 {
return &ColumnInt8{
name: name,
values: values,
}
}
// ColumnInt16 generated columns type for Int16
type ColumnInt16 struct {
ColumnBase
name string
values []int16
}
// Name returns column name
func (c *ColumnInt16) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnInt16) Type() entity.FieldType {
return entity.FieldTypeInt16
}
// Len returns column values length
func (c *ColumnInt16) Len() int {
return len(c.values)
}
func (c *ColumnInt16) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &ColumnInt16{
ColumnBase: c.ColumnBase,
name: c.name,
values: c.values[start:end],
}
}
// Get returns value at index as interface{}.
func (c *ColumnInt16) Get(idx int) (interface{}, error) {
var r int16 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnInt16) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_Int16,
FieldName: c.name,
}
data := make([]int32, 0, c.Len())
for i := 0; i < c.Len(); i++ {
data = append(data, int32(c.values[i]))
}
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: data,
},
},
},
}
return fd
}
// ValueByIdx returns value of the provided index
// error occurs when index out of range
func (c *ColumnInt16) ValueByIdx(idx int) (int16, error) {
var r int16 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnInt16) AppendValue(i interface{}) error {
v, ok := i.(int16)
if !ok {
return fmt.Errorf("invalid type, expected int16, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnInt16) Data() []int16 {
return c.values
}
// NewColumnInt16 auto generated constructor
func NewColumnInt16(name string, values []int16) *ColumnInt16 {
return &ColumnInt16{
name: name,
values: values,
}
}
// ColumnInt32 generated columns type for Int32
type ColumnInt32 struct {
ColumnBase
name string
values []int32
}
// Name returns column name
func (c *ColumnInt32) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnInt32) Type() entity.FieldType {
return entity.FieldTypeInt32
}
// Len returns column values length
func (c *ColumnInt32) Len() int {
return len(c.values)
}
func (c *ColumnInt32) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &ColumnInt32{
ColumnBase: c.ColumnBase,
name: c.name,
values: c.values[start:end],
}
}
// Get returns value at index as interface{}.
func (c *ColumnInt32) Get(idx int) (interface{}, error) {
var r int32 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnInt32) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_Int32,
FieldName: c.name,
}
data := make([]int32, 0, c.Len())
for i := 0; i < c.Len(); i++ {
data = append(data, int32(c.values[i]))
}
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: data,
},
},
},
}
return fd
}
// ValueByIdx returns value of the provided index
// error occurs when index out of range
func (c *ColumnInt32) ValueByIdx(idx int) (int32, error) {
var r int32 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnInt32) AppendValue(i interface{}) error {
v, ok := i.(int32)
if !ok {
return fmt.Errorf("invalid type, expected int32, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnInt32) Data() []int32 {
return c.values
}
// NewColumnInt32 auto generated constructor
func NewColumnInt32(name string, values []int32) *ColumnInt32 {
return &ColumnInt32{
name: name,
values: values,
}
}
// ColumnInt64 generated columns type for Int64
type ColumnInt64 struct {
ColumnBase
name string
values []int64
}
// Name returns column name
func (c *ColumnInt64) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnInt64) Type() entity.FieldType {
return entity.FieldTypeInt64
}
// Len returns column values length
func (c *ColumnInt64) Len() int {
return len(c.values)
}
func (c *ColumnInt64) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &ColumnInt64{
ColumnBase: c.ColumnBase,
name: c.name,
values: c.values[start:end],
}
}
// Get returns value at index as interface{}.
func (c *ColumnInt64) Get(idx int) (interface{}, error) {
var r int64 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnInt64) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_Int64,
FieldName: c.name,
}
data := make([]int64, 0, c.Len())
for i := 0; i < c.Len(); i++ {
data = append(data, int64(c.values[i]))
}
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_LongData{
LongData: &schemapb.LongArray{
Data: data,
},
},
},
}
return fd
}
// ValueByIdx returns value of the provided index
// error occurs when index out of range
func (c *ColumnInt64) ValueByIdx(idx int) (int64, error) {
var r int64 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnInt64) AppendValue(i interface{}) error {
v, ok := i.(int64)
if !ok {
return fmt.Errorf("invalid type, expected int64, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnInt64) Data() []int64 {
return c.values
}
// NewColumnInt64 auto generated constructor
func NewColumnInt64(name string, values []int64) *ColumnInt64 {
return &ColumnInt64{
name: name,
values: values,
}
}
// ColumnFloat generated columns type for Float
type ColumnFloat struct {
ColumnBase
name string
values []float32
}
// Name returns column name
func (c *ColumnFloat) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnFloat) Type() entity.FieldType {
return entity.FieldTypeFloat
}
// Len returns column values length
func (c *ColumnFloat) Len() int {
return len(c.values)
}
func (c *ColumnFloat) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &ColumnFloat{
ColumnBase: c.ColumnBase,
name: c.name,
values: c.values[start:end],
}
}
// Get returns value at index as interface{}.
func (c *ColumnFloat) Get(idx int) (interface{}, error) {
var r float32 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnFloat) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_Float,
FieldName: c.name,
}
data := make([]float32, 0, c.Len())
for i := 0; i < c.Len(); i++ {
data = append(data, float32(c.values[i]))
}
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_FloatData{
FloatData: &schemapb.FloatArray{
Data: data,
},
},
},
}
return fd
}
// ValueByIdx returns value of the provided index
// error occurs when index out of range
func (c *ColumnFloat) ValueByIdx(idx int) (float32, error) {
var r float32 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnFloat) AppendValue(i interface{}) error {
v, ok := i.(float32)
if !ok {
return fmt.Errorf("invalid type, expected float32, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnFloat) Data() []float32 {
return c.values
}
// NewColumnFloat auto generated constructor
func NewColumnFloat(name string, values []float32) *ColumnFloat {
return &ColumnFloat{
name: name,
values: values,
}
}
// ColumnDouble generated columns type for Double
type ColumnDouble struct {
ColumnBase
name string
values []float64
}
// Name returns column name
func (c *ColumnDouble) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnDouble) Type() entity.FieldType {
return entity.FieldTypeDouble
}
// Len returns column values length
func (c *ColumnDouble) Len() int {
return len(c.values)
}
func (c *ColumnDouble) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &ColumnDouble{
ColumnBase: c.ColumnBase,
name: c.name,
values: c.values[start:end],
}
}
// Get returns value at index as interface{}.
func (c *ColumnDouble) Get(idx int) (interface{}, error) {
var r float64 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnDouble) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_Double,
FieldName: c.name,
}
data := make([]float64, 0, c.Len())
for i := 0; i < c.Len(); i++ {
data = append(data, float64(c.values[i]))
}
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_DoubleData{
DoubleData: &schemapb.DoubleArray{
Data: data,
},
},
},
}
return fd
}
// ValueByIdx returns value of the provided index
// error occurs when index out of range
func (c *ColumnDouble) ValueByIdx(idx int) (float64, error) {
var r float64 // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnDouble) AppendValue(i interface{}) error {
v, ok := i.(float64)
if !ok {
return fmt.Errorf("invalid type, expected float64, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnDouble) Data() []float64 {
return c.values
}
// NewColumnDouble auto generated constructor
func NewColumnDouble(name string, values []float64) *ColumnDouble {
return &ColumnDouble{
name: name,
values: values,
}
}
// ColumnString generated columns type for String
type ColumnString struct {
ColumnBase
name string
values []string
}
// Name returns column name
func (c *ColumnString) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnString) Type() entity.FieldType {
return entity.FieldTypeString
}
// Len returns column values length
func (c *ColumnString) Len() int {
return len(c.values)
}
func (c *ColumnString) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &ColumnString{
ColumnBase: c.ColumnBase,
name: c.name,
values: c.values[start:end],
}
}
// Get returns value at index as interface{}.
func (c *ColumnString) Get(idx int) (interface{}, error) {
var r string // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnString) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_String,
FieldName: c.name,
}
data := make([]string, 0, c.Len())
for i := 0; i < c.Len(); i++ {
data = append(data, string(c.values[i]))
}
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_StringData{
StringData: &schemapb.StringArray{
Data: data,
},
},
},
}
return fd
}
// ValueByIdx returns value of the provided index
// error occurs when index out of range
func (c *ColumnString) ValueByIdx(idx int) (string, error) {
var r string // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnString) AppendValue(i interface{}) error {
v, ok := i.(string)
if !ok {
return fmt.Errorf("invalid type, expected string, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnString) Data() []string {
return c.values
}
// NewColumnString auto generated constructor
func NewColumnString(name string, values []string) *ColumnString {
return &ColumnString{
name: name,
values: values,
}
}

855
client/column/scalar_gen_test.go
View File

@ -1,855 +0,0 @@
// Code generated by go generate; DO NOT EDIT
// This file is generated by go generated
package column
import (
"fmt"
"math/rand"
"testing"
"time"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/stretchr/testify/assert"
)
func TestColumnBool(t *testing.T) {
rand.Seed(time.Now().UnixNano())
columnName := fmt.Sprintf("column_Bool_%d", rand.Int())
columnLen := 8 + rand.Intn(10)
v := make([]bool, columnLen)
column := NewColumnBool(columnName, v)
t.Run("test meta", func(t *testing.T) {
ft := entity.FieldTypeBool
assert.Equal(t, "Bool", ft.Name())
assert.Equal(t, "bool", ft.String())
pbName, pbType := ft.PbFieldType()
assert.Equal(t, "Bool", pbName)
assert.Equal(t, "bool", pbType)
})
t.Run("test column attribute", func(t *testing.T) {
assert.Equal(t, columnName, column.Name())
assert.Equal(t, entity.FieldTypeBool, column.Type())
assert.Equal(t, columnLen, column.Len())
assert.EqualValues(t, v, column.Data())
})
t.Run("test column field data", func(t *testing.T) {
fd := column.FieldData()
assert.NotNil(t, fd)
assert.Equal(t, fd.GetFieldName(), columnName)
})
t.Run("test column value by idx", func(t *testing.T) {
_, err := column.ValueByIdx(-1)
assert.NotNil(t, err)
_, err = column.ValueByIdx(columnLen)
assert.NotNil(t, err)
for i := 0; i < columnLen; i++ {
v, err := column.ValueByIdx(i)
assert.Nil(t, err)
assert.Equal(t, column.values[i], v)
}
})
}
func TestFieldDataBoolColumn(t *testing.T) {
len := rand.Intn(10) + 8
name := fmt.Sprintf("fd_Bool_%d", rand.Int())
fd := &schemapb.FieldData{
Type: schemapb.DataType_Bool,
FieldName: name,
}
t.Run("normal usage", func(t *testing.T) {
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_BoolData{
BoolData: &schemapb.BoolArray{
Data: make([]bool, len),
},
},
},
}
column, err := FieldDataColumn(fd, 0, len)
assert.Nil(t, err)
assert.NotNil(t, column)
assert.Equal(t, name, column.Name())
assert.Equal(t, len, column.Len())
assert.Equal(t, entity.FieldTypeBool, column.Type())
var ev bool
err = column.AppendValue(ev)
assert.Equal(t, len+1, column.Len())
assert.Nil(t, err)
err = column.AppendValue(struct{}{})
assert.Equal(t, len+1, column.Len())
assert.NotNil(t, err)
})
t.Run("nil data", func(t *testing.T) {
fd.Field = nil
_, err := FieldDataColumn(fd, 0, len)
assert.NotNil(t, err)
})
t.Run("get all data", func(t *testing.T) {
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_BoolData{
BoolData: &schemapb.BoolArray{
Data: make([]bool, len),
},
},
},
}
column, err := FieldDataColumn(fd, 0, -1)
assert.Nil(t, err)
assert.NotNil(t, column)
assert.Equal(t, name, column.Name())
assert.Equal(t, len, column.Len())
assert.Equal(t, entity.FieldTypeBool, column.Type())
})
}
func TestColumnInt8(t *testing.T) {
rand.Seed(time.Now().UnixNano())
columnName := fmt.Sprintf("column_Int8_%d", rand.Int())
columnLen := 8 + rand.Intn(10)
v := make([]int8, columnLen)
column := NewColumnInt8(columnName, v)
t.Run("test meta", func(t *testing.T) {
ft := entity.FieldTypeInt8
assert.Equal(t, "Int8", ft.Name())
assert.Equal(t, "int8", ft.String())
pbName, pbType := ft.PbFieldType()
assert.Equal(t, "Int", pbName)
assert.Equal(t, "int32", pbType)
})
t.Run("test column attribute", func(t *testing.T) {
assert.Equal(t, columnName, column.Name())
assert.Equal(t, entity.FieldTypeInt8, column.Type())
assert.Equal(t, columnLen, column.Len())
assert.EqualValues(t, v, column.Data())
})
t.Run("test column field data", func(t *testing.T) {
fd := column.FieldData()
assert.NotNil(t, fd)
assert.Equal(t, fd.GetFieldName(), columnName)
})
t.Run("test column value by idx", func(t *testing.T) {
_, err := column.ValueByIdx(-1)
assert.NotNil(t, err)
_, err = column.ValueByIdx(columnLen)
assert.NotNil(t, err)
for i := 0; i < columnLen; i++ {
v, err := column.ValueByIdx(i)
assert.Nil(t, err)
assert.Equal(t, column.values[i], v)
}
})
}
func TestFieldDataInt8Column(t *testing.T) {
len := rand.Intn(10) + 8
name := fmt.Sprintf("fd_Int8_%d", rand.Int())
fd := &schemapb.FieldData{
Type: schemapb.DataType_Int8,
FieldName: name,
}
t.Run("normal usage", func(t *testing.T) {
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: make([]int32, len),
},
},
},
}
column, err := FieldDataColumn(fd, 0, len)
assert.Nil(t, err)
assert.NotNil(t, column)
assert.Equal(t, name, column.Name())
assert.Equal(t, len, column.Len())
assert.Equal(t, entity.FieldTypeInt8, column.Type())
var ev int8
err = column.AppendValue(ev)
assert.Equal(t, len+1, column.Len())
assert.Nil(t, err)
err = column.AppendValue(struct{}{})
assert.Equal(t, len+1, column.Len())
assert.NotNil(t, err)
})
t.Run("nil data", func(t *testing.T) {
fd.Field = nil
_, err := FieldDataColumn(fd, 0, len)
assert.NotNil(t, err)
})
t.Run("get all data", func(t *testing.T) {
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: make([]int32, len),
},
},
},
}
column, err := FieldDataColumn(fd, 0, -1)
assert.Nil(t, err)
assert.NotNil(t, column)
assert.Equal(t, name, column.Name())
assert.Equal(t, len, column.Len())
assert.Equal(t, entity.FieldTypeInt8, column.Type())
})
}
func TestColumnInt16(t *testing.T) {
rand.Seed(time.Now().UnixNano())
columnName := fmt.Sprintf("column_Int16_%d", rand.Int())
columnLen := 8 + rand.Intn(10)
v := make([]int16, columnLen)
column := NewColumnInt16(columnName, v)
t.Run("test meta", func(t *testing.T) {
ft := entity.FieldTypeInt16
assert.Equal(t, "Int16", ft.Name())
assert.Equal(t, "int16", ft.String())
pbName, pbType := ft.PbFieldType()
assert.Equal(t, "Int", pbName)
assert.Equal(t, "int32", pbType)
})
t.Run("test column attribute", func(t *testing.T) {
assert.Equal(t, columnName, column.Name())
assert.Equal(t, entity.FieldTypeInt16, column.Type())
assert.Equal(t, columnLen, column.Len())
assert.EqualValues(t, v, column.Data())
})
t.Run("test column field data", func(t *testing.T) {
fd := column.FieldData()
assert.NotNil(t, fd)
assert.Equal(t, fd.GetFieldName(), columnName)
})
t.Run("test column value by idx", func(t *testing.T) {
_, err := column.ValueByIdx(-1)
assert.NotNil(t, err)
_, err = column.ValueByIdx(columnLen)
assert.NotNil(t, err)
for i := 0; i < columnLen; i++ {
v, err := column.ValueByIdx(i)
assert.Nil(t, err)
assert.Equal(t, column.values[i], v)
}
})
}
func TestFieldDataInt16Column(t *testing.T) {
len := rand.Intn(10) + 8
name := fmt.Sprintf("fd_Int16_%d", rand.Int())
fd := &schemapb.FieldData{
Type: schemapb.DataType_Int16,
FieldName: name,
}
t.Run("normal usage", func(t *testing.T) {
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: make([]int32, len),
},
},
},
}
column, err := FieldDataColumn(fd, 0, len)
assert.Nil(t, err)
assert.NotNil(t, column)
assert.Equal(t, name, column.Name())
assert.Equal(t, len, column.Len())
assert.Equal(t, entity.FieldTypeInt16, column.Type())
var ev int16
err = column.AppendValue(ev)
assert.Equal(t, len+1, column.Len())
assert.Nil(t, err)
err = column.AppendValue(struct{}{})
assert.Equal(t, len+1, column.Len())
assert.NotNil(t, err)
})
t.Run("nil data", func(t *testing.T) {
fd.Field = nil
_, err := FieldDataColumn(fd, 0, len)
assert.NotNil(t, err)
})
t.Run("get all data", func(t *testing.T) {
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: make([]int32, len),
},
},
},
}
column, err := FieldDataColumn(fd, 0, -1)
assert.Nil(t, err)
assert.NotNil(t, column)
assert.Equal(t, name, column.Name())
assert.Equal(t, len, column.Len())
assert.Equal(t, entity.FieldTypeInt16, column.Type())
})
}
func TestColumnInt32(t *testing.T) {
rand.Seed(time.Now().UnixNano())
columnName := fmt.Sprintf("column_Int32_%d", rand.Int())
columnLen := 8 + rand.Intn(10)
v := make([]int32, columnLen)
column := NewColumnInt32(columnName, v)
t.Run("test meta", func(t *testing.T) {
ft := entity.FieldTypeInt32
assert.Equal(t, "Int32", ft.Name())
assert.Equal(t, "int32", ft.String())
pbName, pbType := ft.PbFieldType()
assert.Equal(t, "Int", pbName)
assert.Equal(t, "int32", pbType)
})
t.Run("test column attribute", func(t *testing.T) {
assert.Equal(t, columnName, column.Name())
assert.Equal(t, entity.FieldTypeInt32, column.Type())
assert.Equal(t, columnLen, column.Len())
assert.EqualValues(t, v, column.Data())
})
t.Run("test column field data", func(t *testing.T) {
fd := column.FieldData()
assert.NotNil(t, fd)
assert.Equal(t, fd.GetFieldName(), columnName)
})
t.Run("test column value by idx", func(t *testing.T) {
_, err := column.ValueByIdx(-1)
assert.NotNil(t, err)
_, err = column.ValueByIdx(columnLen)
assert.NotNil(t, err)
for i := 0; i < columnLen; i++ {
v, err := column.ValueByIdx(i)
assert.Nil(t, err)
assert.Equal(t, column.values[i], v)
}
})
}
func TestFieldDataInt32Column(t *testing.T) {
len := rand.Intn(10) + 8
name := fmt.Sprintf("fd_Int32_%d", rand.Int())
fd := &schemapb.FieldData{
Type: schemapb.DataType_Int32,
FieldName: name,
}
t.Run("normal usage", func(t *testing.T) {
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: make([]int32, len),
},
},
},
}
column, err := FieldDataColumn(fd, 0, len)
assert.Nil(t, err)
assert.NotNil(t, column)
assert.Equal(t, name, column.Name())
assert.Equal(t, len, column.Len())
assert.Equal(t, entity.FieldTypeInt32, column.Type())
var ev int32
err = column.AppendValue(ev)
assert.Equal(t, len+1, column.Len())
assert.Nil(t, err)
err = column.AppendValue(struct{}{})
assert.Equal(t, len+1, column.Len())
assert.NotNil(t, err)
})
t.Run("nil data", func(t *testing.T) {
fd.Field = nil
_, err := FieldDataColumn(fd, 0, len)
assert.NotNil(t, err)
})
t.Run("get all data", func(t *testing.T) {
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: make([]int32, len),
},
},
},
}
column, err := FieldDataColumn(fd, 0, -1)
assert.Nil(t, err)
assert.NotNil(t, column)
assert.Equal(t, name, column.Name())
assert.Equal(t, len, column.Len())
assert.Equal(t, entity.FieldTypeInt32, column.Type())
})
}
func TestColumnInt64(t *testing.T) {
rand.Seed(time.Now().UnixNano())
columnName := fmt.Sprintf("column_Int64_%d", rand.Int())
columnLen := 8 + rand.Intn(10)
v := make([]int64, columnLen)
column := NewColumnInt64(columnName, v)
t.Run("test meta", func(t *testing.T) {
ft := entity.FieldTypeInt64
assert.Equal(t, "Int64", ft.Name())
assert.Equal(t, "int64", ft.String())
pbName, pbType := ft.PbFieldType()
assert.Equal(t, "Long", pbName)
assert.Equal(t, "int64", pbType)
})
t.Run("test column attribute", func(t *testing.T) {
assert.Equal(t, columnName, column.Name())
assert.Equal(t, entity.FieldTypeInt64, column.Type())
assert.Equal(t, columnLen, column.Len())
assert.EqualValues(t, v, column.Data())
})
t.Run("test column field data", func(t *testing.T) {
fd := column.FieldData()
assert.NotNil(t, fd)
assert.Equal(t, fd.GetFieldName(), columnName)
})
t.Run("test column value by idx", func(t *testing.T) {
_, err := column.ValueByIdx(-1)
assert.NotNil(t, err)
_, err = column.ValueByIdx(columnLen)
assert.NotNil(t, err)
for i := 0; i < columnLen; i++ {
v, err := column.ValueByIdx(i)
assert.Nil(t, err)
assert.Equal(t, column.values[i], v)
}
})
}
func TestFieldDataInt64Column(t *testing.T) {
len := rand.Intn(10) + 8
name := fmt.Sprintf("fd_Int64_%d", rand.Int())
fd := &schemapb.FieldData{
Type: schemapb.DataType_Int64,
FieldName: name,
}
t.Run("normal usage", func(t *testing.T) {
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_LongData{
LongData: &schemapb.LongArray{
Data: make([]int64, len),
},
},
},
}
column, err := FieldDataColumn(fd, 0, len)
assert.Nil(t, err)
assert.NotNil(t, column)
assert.Equal(t, name, column.Name())
assert.Equal(t, len, column.Len())
assert.Equal(t, entity.FieldTypeInt64, column.Type())
var ev int64
err = column.AppendValue(ev)
assert.Equal(t, len+1, column.Len())
assert.Nil(t, err)
err = column.AppendValue(struct{}{})
assert.Equal(t, len+1, column.Len())
assert.NotNil(t, err)
})
t.Run("nil data", func(t *testing.T) {
fd.Field = nil
_, err := FieldDataColumn(fd, 0, len)
assert.NotNil(t, err)
})
t.Run("get all data", func(t *testing.T) {
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_LongData{
LongData: &schemapb.LongArray{
Data: make([]int64, len),
},
},
},
}
column, err := FieldDataColumn(fd, 0, -1)
assert.Nil(t, err)
assert.NotNil(t, column)
assert.Equal(t, name, column.Name())
assert.Equal(t, len, column.Len())
assert.Equal(t, entity.FieldTypeInt64, column.Type())
})
}
func TestColumnFloat(t *testing.T) {
rand.Seed(time.Now().UnixNano())
columnName := fmt.Sprintf("column_Float_%d", rand.Int())
columnLen := 8 + rand.Intn(10)
v := make([]float32, columnLen)
column := NewColumnFloat(columnName, v)
t.Run("test meta", func(t *testing.T) {
ft := entity.FieldTypeFloat
assert.Equal(t, "Float", ft.Name())
assert.Equal(t, "float32", ft.String())
pbName, pbType := ft.PbFieldType()
assert.Equal(t, "Float", pbName)
assert.Equal(t, "float32", pbType)
})
t.Run("test column attribute", func(t *testing.T) {
assert.Equal(t, columnName, column.Name())
assert.Equal(t, entity.FieldTypeFloat, column.Type())
assert.Equal(t, columnLen, column.Len())
assert.EqualValues(t, v, column.Data())
})
t.Run("test column field data", func(t *testing.T) {
fd := column.FieldData()
assert.NotNil(t, fd)
assert.Equal(t, fd.GetFieldName(), columnName)
})
t.Run("test column value by idx", func(t *testing.T) {
_, err := column.ValueByIdx(-1)
assert.NotNil(t, err)
_, err = column.ValueByIdx(columnLen)
assert.NotNil(t, err)
for i := 0; i < columnLen; i++ {
v, err := column.ValueByIdx(i)
assert.Nil(t, err)
assert.Equal(t, column.values[i], v)
}
})
}
func TestFieldDataFloatColumn(t *testing.T) {
len := rand.Intn(10) + 8
name := fmt.Sprintf("fd_Float_%d", rand.Int())
fd := &schemapb.FieldData{
Type: schemapb.DataType_Float,
FieldName: name,
}
t.Run("normal usage", func(t *testing.T) {
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_FloatData{
FloatData: &schemapb.FloatArray{
Data: make([]float32, len),
},
},
},
}
column, err := FieldDataColumn(fd, 0, len)
assert.Nil(t, err)
assert.NotNil(t, column)
assert.Equal(t, name, column.Name())
assert.Equal(t, len, column.Len())
assert.Equal(t, entity.FieldTypeFloat, column.Type())
var ev float32
err = column.AppendValue(ev)
assert.Equal(t, len+1, column.Len())
assert.Nil(t, err)
err = column.AppendValue(struct{}{})
assert.Equal(t, len+1, column.Len())
assert.NotNil(t, err)
})
t.Run("nil data", func(t *testing.T) {
fd.Field = nil
_, err := FieldDataColumn(fd, 0, len)
assert.NotNil(t, err)
})
t.Run("get all data", func(t *testing.T) {
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_FloatData{
FloatData: &schemapb.FloatArray{
Data: make([]float32, len),
},
},
},
}
column, err := FieldDataColumn(fd, 0, -1)
assert.Nil(t, err)
assert.NotNil(t, column)
assert.Equal(t, name, column.Name())
assert.Equal(t, len, column.Len())
assert.Equal(t, entity.FieldTypeFloat, column.Type())
})
}
func TestColumnDouble(t *testing.T) {
rand.Seed(time.Now().UnixNano())
columnName := fmt.Sprintf("column_Double_%d", rand.Int())
columnLen := 8 + rand.Intn(10)
v := make([]float64, columnLen)
column := NewColumnDouble(columnName, v)
t.Run("test meta", func(t *testing.T) {
ft := entity.FieldTypeDouble
assert.Equal(t, "Double", ft.Name())
assert.Equal(t, "float64", ft.String())
pbName, pbType := ft.PbFieldType()
assert.Equal(t, "Double", pbName)
assert.Equal(t, "float64", pbType)
})
t.Run("test column attribute", func(t *testing.T) {
assert.Equal(t, columnName, column.Name())
assert.Equal(t, entity.FieldTypeDouble, column.Type())
assert.Equal(t, columnLen, column.Len())
assert.EqualValues(t, v, column.Data())
})
t.Run("test column field data", func(t *testing.T) {
fd := column.FieldData()
assert.NotNil(t, fd)
assert.Equal(t, fd.GetFieldName(), columnName)
})
t.Run("test column value by idx", func(t *testing.T) {
_, err := column.ValueByIdx(-1)
assert.NotNil(t, err)
_, err = column.ValueByIdx(columnLen)
assert.NotNil(t, err)
for i := 0; i < columnLen; i++ {
v, err := column.ValueByIdx(i)
assert.Nil(t, err)
assert.Equal(t, column.values[i], v)
}
})
}
func TestFieldDataDoubleColumn(t *testing.T) {
len := rand.Intn(10) + 8
name := fmt.Sprintf("fd_Double_%d", rand.Int())
fd := &schemapb.FieldData{
Type: schemapb.DataType_Double,
FieldName: name,
}
t.Run("normal usage", func(t *testing.T) {
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_DoubleData{
DoubleData: &schemapb.DoubleArray{
Data: make([]float64, len),
},
},
},
}
column, err := FieldDataColumn(fd, 0, len)
assert.Nil(t, err)
assert.NotNil(t, column)
assert.Equal(t, name, column.Name())
assert.Equal(t, len, column.Len())
assert.Equal(t, entity.FieldTypeDouble, column.Type())
var ev float64
err = column.AppendValue(ev)
assert.Equal(t, len+1, column.Len())
assert.Nil(t, err)
err = column.AppendValue(struct{}{})
assert.Equal(t, len+1, column.Len())
assert.NotNil(t, err)
})
t.Run("nil data", func(t *testing.T) {
fd.Field = nil
_, err := FieldDataColumn(fd, 0, len)
assert.NotNil(t, err)
})
t.Run("get all data", func(t *testing.T) {
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_DoubleData{
DoubleData: &schemapb.DoubleArray{
Data: make([]float64, len),
},
},
},
}
column, err := FieldDataColumn(fd, 0, -1)
assert.Nil(t, err)
assert.NotNil(t, column)
assert.Equal(t, name, column.Name())
assert.Equal(t, len, column.Len())
assert.Equal(t, entity.FieldTypeDouble, column.Type())
})
}
func TestColumnString(t *testing.T) {
rand.Seed(time.Now().UnixNano())
columnName := fmt.Sprintf("column_String_%d", rand.Int())
columnLen := 8 + rand.Intn(10)
v := make([]string, columnLen)
column := NewColumnString(columnName, v)
t.Run("test meta", func(t *testing.T) {
ft := entity.FieldTypeString
assert.Equal(t, "String", ft.Name())
assert.Equal(t, "string", ft.String())
pbName, pbType := ft.PbFieldType()
assert.Equal(t, "String", pbName)
assert.Equal(t, "string", pbType)
})
t.Run("test column attribute", func(t *testing.T) {
assert.Equal(t, columnName, column.Name())
assert.Equal(t, entity.FieldTypeString, column.Type())
assert.Equal(t, columnLen, column.Len())
assert.EqualValues(t, v, column.Data())
})
t.Run("test column field data", func(t *testing.T) {
fd := column.FieldData()
assert.NotNil(t, fd)
assert.Equal(t, fd.GetFieldName(), columnName)
})
t.Run("test column value by idx", func(t *testing.T) {
_, err := column.ValueByIdx(-1)
assert.NotNil(t, err)
_, err = column.ValueByIdx(columnLen)
assert.NotNil(t, err)
for i := 0; i < columnLen; i++ {
v, err := column.ValueByIdx(i)
assert.Nil(t, err)
assert.Equal(t, column.values[i], v)
}
})
}
func TestFieldDataStringColumn(t *testing.T) {
len := rand.Intn(10) + 8
name := fmt.Sprintf("fd_String_%d", rand.Int())
fd := &schemapb.FieldData{
Type: schemapb.DataType_String,
FieldName: name,
}
t.Run("normal usage", func(t *testing.T) {
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_StringData{
StringData: &schemapb.StringArray{
Data: make([]string, len),
},
},
},
}
column, err := FieldDataColumn(fd, 0, len)
assert.Nil(t, err)
assert.NotNil(t, column)
assert.Equal(t, name, column.Name())
assert.Equal(t, len, column.Len())
assert.Equal(t, entity.FieldTypeString, column.Type())
var ev string
err = column.AppendValue(ev)
assert.Equal(t, len+1, column.Len())
assert.Nil(t, err)
err = column.AppendValue(struct{}{})
assert.Equal(t, len+1, column.Len())
assert.NotNil(t, err)
})
t.Run("nil data", func(t *testing.T) {
fd.Field = nil
_, err := FieldDataColumn(fd, 0, len)
assert.NotNil(t, err)
})
t.Run("get all data", func(t *testing.T) {
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_StringData{
StringData: &schemapb.StringArray{
Data: make([]string, len),
},
},
},
}
column, err := FieldDataColumn(fd, 0, -1)
assert.Nil(t, err)
assert.NotNil(t, column)
assert.Equal(t, name, column.Name())
assert.Equal(t, len, column.Len())
assert.Equal(t, entity.FieldTypeString, column.Type())
})
}

219
client/column/scalar_test.go Normal file
View File

@ -0,0 +1,219 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package column
import (
"fmt"
"math/rand"
"testing"
"github.com/stretchr/testify/suite"
"github.com/milvus-io/milvus/client/v2/entity"
)
type ScalarSuite struct {
suite.Suite
}
func (s *ScalarSuite) TestBasic() {
s.Run("column_bool", func() {
name := fmt.Sprintf("field_%d", rand.Intn(1000))
data := []bool{true, false}
column := NewColumnBool(name, data)
s.Equal(entity.FieldTypeBool, column.Type())
s.Equal(name, column.Name())
s.Equal(data, column.Data())
fd := column.FieldData()
s.Equal(name, fd.GetFieldName())
s.Equal(data, fd.GetScalars().GetBoolData().GetData())
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnBool)
if s.True(ok) {
s.Equal(name, parsed.Name())
s.Equal(data, parsed.Data())
s.Equal(entity.FieldTypeBool, column.Type())
}
})
s.Run("column_int8", func() {
name := fmt.Sprintf("field_%d", rand.Intn(1000))
data := []int8{1, 2, 3}
column := NewColumnInt8(name, data)
s.Equal(entity.FieldTypeInt8, column.Type())
s.Equal(name, column.Name())
s.Equal(data, column.Data())
fd := column.FieldData()
s.Equal(name, fd.GetFieldName())
fdData := fd.GetScalars().GetIntData().GetData()
for i, row := range data {
s.EqualValues(row, fdData[i])
}
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnInt8)
if s.True(ok) {
s.Equal(name, parsed.Name())
s.Equal(data, parsed.Data())
s.Equal(entity.FieldTypeInt8, column.Type())
}
})
s.Run("column_int16", func() {
name := fmt.Sprintf("field_%d", rand.Intn(1000))
data := []int16{1, 2, 3}
column := NewColumnInt16(name, data)
s.Equal(entity.FieldTypeInt16, column.Type())
s.Equal(name, column.Name())
s.Equal(data, column.Data())
fd := column.FieldData()
s.Equal(name, fd.GetFieldName())
fdData := fd.GetScalars().GetIntData().GetData()
for i, row := range data {
s.EqualValues(row, fdData[i])
}
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnInt16)
if s.True(ok) {
s.Equal(name, parsed.Name())
s.Equal(data, parsed.Data())
s.Equal(entity.FieldTypeInt16, column.Type())
}
})
s.Run("column_int32", func() {
name := fmt.Sprintf("field_%d", rand.Intn(1000))
data := []int32{1, 2, 3}
column := NewColumnInt32(name, data)
s.Equal(entity.FieldTypeInt32, column.Type())
s.Equal(name, column.Name())
s.Equal(data, column.Data())
fd := column.FieldData()
s.Equal(name, fd.GetFieldName())
s.Equal(data, fd.GetScalars().GetIntData().GetData())
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnInt32)
if s.True(ok) {
s.Equal(name, parsed.Name())
s.Equal(data, parsed.Data())
s.Equal(entity.FieldTypeInt32, column.Type())
}
})
s.Run("column_int64", func() {
name := fmt.Sprintf("field_%d", rand.Intn(1000))
data := []int64{1, 2, 3}
column := NewColumnInt64(name, data)
s.Equal(entity.FieldTypeInt64, column.Type())
s.Equal(name, column.Name())
s.Equal(data, column.Data())
fd := column.FieldData()
s.Equal(name, fd.GetFieldName())
s.Equal(data, fd.GetScalars().GetLongData().GetData())
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnInt64)
if s.True(ok) {
s.Equal(name, parsed.Name())
s.Equal(data, parsed.Data())
s.Equal(entity.FieldTypeInt64, column.Type())
}
})
s.Run("column_float", func() {
name := fmt.Sprintf("field_%d", rand.Intn(1000))
data := []float32{1.1, 2.2, 3.3}
column := NewColumnFloat(name, data)
s.Equal(entity.FieldTypeFloat, column.Type())
s.Equal(name, column.Name())
s.Equal(data, column.Data())
fd := column.FieldData()
s.Equal(name, fd.GetFieldName())
s.Equal(data, fd.GetScalars().GetFloatData().GetData())
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnFloat)
if s.True(ok) {
s.Equal(name, parsed.Name())
s.Equal(data, parsed.Data())
s.Equal(entity.FieldTypeFloat, column.Type())
}
})
s.Run("column_double", func() {
name := fmt.Sprintf("field_%d", rand.Intn(1000))
data := []float64{1.1, 2.2, 3.3}
column := NewColumnDouble(name, data)
s.Equal(entity.FieldTypeDouble, column.Type())
s.Equal(name, column.Name())
s.Equal(data, column.Data())
fd := column.FieldData()
s.Equal(name, fd.GetFieldName())
s.Equal(data, fd.GetScalars().GetDoubleData().GetData())
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnDouble)
if s.True(ok) {
s.Equal(name, parsed.Name())
s.Equal(data, parsed.Data())
s.Equal(entity.FieldTypeDouble, column.Type())
}
})
s.Run("column_varchar", func() {
name := fmt.Sprintf("field_%d", rand.Intn(1000))
data := []string{"a", "b", "c"}
column := NewColumnVarChar(name, data)
s.Equal(entity.FieldTypeVarChar, column.Type())
s.Equal(name, column.Name())
s.Equal(data, column.Data())
fd := column.FieldData()
s.Equal(name, fd.GetFieldName())
s.Equal(data, fd.GetScalars().GetStringData().GetData())
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnVarChar)
if s.True(ok) {
s.Equal(name, parsed.Name())
s.Equal(data, parsed.Data())
s.Equal(entity.FieldTypeVarChar, column.Type())
}
})
}
func TestScalarColumn(t *testing.T) {
suite.Run(t, new(ScalarSuite))
}

122
client/column/sparse.go
View File

@ -17,11 +17,7 @@
package column
import (
"encoding/binary"
"fmt"
"math"
"github.com/cockroachdb/errors"
"github.com/samber/lo"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/client/v2/entity"
@ -30,112 +26,22 @@ import (
var _ (Column) = (*ColumnSparseFloatVector)(nil)
type ColumnSparseFloatVector struct {
ColumnBase
vectors []entity.SparseEmbedding
name string
}
// Name returns column name.
func (c *ColumnSparseFloatVector) Name() string {
return c.name
}
// Type returns column FieldType.
func (c *ColumnSparseFloatVector) Type() entity.FieldType {
return entity.FieldTypeSparseVector
}
// Len returns column values length.
func (c *ColumnSparseFloatVector) Len() int {
return len(c.vectors)
}
func (c *ColumnSparseFloatVector) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &ColumnSparseFloatVector{
ColumnBase: c.ColumnBase,
name: c.name,
vectors: c.vectors[start:end],
}
}
// Get returns value at index as interface{}.
func (c *ColumnSparseFloatVector) Get(idx int) (interface{}, error) {
if idx < 0 || idx >= c.Len() {
return nil, errors.New("index out of range")
}
return c.vectors[idx], nil
}
// ValueByIdx returns value of the provided index
// error occurs when index out of range
func (c *ColumnSparseFloatVector) ValueByIdx(idx int) (entity.SparseEmbedding, error) {
var r entity.SparseEmbedding // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.vectors[idx], nil
}
func (c *ColumnSparseFloatVector) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_SparseFloatVector,
FieldName: c.name,
}
dim := int(0)
data := make([][]byte, 0, len(c.vectors))
for _, vector := range c.vectors {
row := make([]byte, 8*vector.Len())
for idx := 0; idx < vector.Len(); idx++ {
pos, value, _ := vector.Get(idx)
binary.LittleEndian.PutUint32(row[idx*8:], pos)
binary.LittleEndian.PutUint32(row[idx*8+4:], math.Float32bits(value))
}
data = append(data, row)
if vector.Dim() > dim {
dim = vector.Dim()
}
}
fd.Field = &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: int64(dim),
Data: &schemapb.VectorField_SparseFloatVector{
SparseFloatVector: &schemapb.SparseFloatArray{
Dim: int64(dim),
Contents: data,
},
},
},
}
return fd
}
func (c *ColumnSparseFloatVector) AppendValue(i interface{}) error {
v, ok := i.(entity.SparseEmbedding)
if !ok {
return fmt.Errorf("invalid type, expect SparseEmbedding interface, got %T", i)
}
c.vectors = append(c.vectors, v)
return nil
}
func (c *ColumnSparseFloatVector) Data() []entity.SparseEmbedding {
return c.vectors
*vectorBase[entity.SparseEmbedding]
}
func NewColumnSparseVectors(name string, values []entity.SparseEmbedding) *ColumnSparseFloatVector {
return &ColumnSparseFloatVector{
name: name,
vectors: values,
// sparse embedding need not specify dimension
vectorBase: newVectorBase(name, 0, values, entity.FieldTypeSparseVector),
}
}
func (c *ColumnSparseFloatVector) FieldData() *schemapb.FieldData {
fd := c.vectorBase.FieldData()
max := lo.MaxBy(c.values, func(a, b entity.SparseEmbedding) bool {
return a.Dim() > b.Dim()
})
vectors := fd.GetVectors()
vectors.Dim = int64(max.Dim())
return fd
}

20
client/column/sparse_test.go
View File

@ -57,12 +57,24 @@ func TestColumnSparseEmbedding(t *testing.T) {
fd := column.FieldData()
assert.NotNil(t, fd)
assert.Equal(t, fd.GetFieldName(), columnName)
result, err := FieldDataColumn(fd, 0, -1)
assert.NoError(t, err)
parsed, ok := result.(*ColumnSparseFloatVector)
if assert.True(t, ok) {
assert.Equal(t, columnName, parsed.Name())
assert.Equal(t, entity.FieldTypeSparseVector, parsed.Type())
assert.Equal(t, columnLen, parsed.Len())
// dim not equal
// assert.EqualValues(t, v, parsed.Data())
}
})
t.Run("test column value by idx", func(t *testing.T) {
_, err := column.ValueByIdx(-1)
_, err := column.Value(-1)
assert.Error(t, err)
_, err = column.ValueByIdx(columnLen)
_, err = column.Value(columnLen)
assert.Error(t, err)
_, err = column.Get(-1)
@ -71,9 +83,9 @@ func TestColumnSparseEmbedding(t *testing.T) {
assert.Error(t, err)
for i := 0; i < columnLen; i++ {
v, err := column.ValueByIdx(i)
v, err := column.Value(i)
assert.NoError(t, err)
assert.Equal(t, column.vectors[i], v)
assert.Equal(t, column.values[i], v)
getV, err := column.Get(i)
assert.NoError(t, err)
assert.Equal(t, v, getV)

135
client/column/varchar.go
View File

@ -1,135 +0,0 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package column
import (
"fmt"
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/client/v2/entity"
)
// ColumnVarChar generated columns type for VarChar
type ColumnVarChar struct {
ColumnBase
name string
values []string
}
// Name returns column name
func (c *ColumnVarChar) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnVarChar) Type() entity.FieldType {
return entity.FieldTypeVarChar
}
// Len returns column values length
func (c *ColumnVarChar) Len() int {
return len(c.values)
}
func (c *ColumnVarChar) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
return &ColumnVarChar{
ColumnBase: c.ColumnBase,
name: c.name,
values: c.values[start:end],
}
}
// Get returns value at index as interface{}.
func (c *ColumnVarChar) Get(idx int) (interface{}, error) {
if idx < 0 || idx > c.Len() {
return "", errors.New("index out of range")
}
return c.values[idx], nil
}
// GetAsString returns value at idx.
func (c *ColumnVarChar) GetAsString(idx int) (string, error) {
if idx < 0 || idx > c.Len() {
return "", errors.New("index out of range")
}
return c.values[idx], nil
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnVarChar) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_VarChar,
FieldName: c.name,
}
data := make([]string, 0, c.Len())
for i := 0; i < c.Len(); i++ {
data = append(data, c.values[i])
}
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_StringData{
StringData: &schemapb.StringArray{
Data: data,
},
},
},
}
return fd
}
// ValueByIdx returns value of the provided index
// error occurs when index out of range
func (c *ColumnVarChar) ValueByIdx(idx int) (string, error) {
var r string // use default value
if idx < 0 || idx >= c.Len() {
return r, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnVarChar) AppendValue(i interface{}) error {
v, ok := i.(string)
if !ok {
return fmt.Errorf("invalid type, expected string, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnVarChar) Data() []string {
return c.values
}
// NewColumnVarChar auto generated constructor
func NewColumnVarChar(name string, values []string) *ColumnVarChar {
return &ColumnVarChar{
name: name,
values: values,
}
}

134
client/column/varchar_test.go
View File

@ -1,134 +0,0 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package column
import (
"fmt"
"math/rand"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/client/v2/entity"
)
func TestColumnVarChar(t *testing.T) {
rand.Seed(time.Now().UnixNano())
columnName := fmt.Sprintf("column_VarChar_%d", rand.Int())
columnLen := 8 + rand.Intn(10)
v := make([]string, columnLen)
column := NewColumnVarChar(columnName, v)
t.Run("test meta", func(t *testing.T) {
ft := entity.FieldTypeVarChar
assert.Equal(t, "VarChar", ft.Name())
assert.Equal(t, "string", ft.String())
pbName, pbType := ft.PbFieldType()
assert.Equal(t, "VarChar", pbName)
assert.Equal(t, "string", pbType)
})
t.Run("test column attribute", func(t *testing.T) {
assert.Equal(t, columnName, column.Name())
assert.Equal(t, entity.FieldTypeVarChar, column.Type())
assert.Equal(t, columnLen, column.Len())
assert.EqualValues(t, v, column.Data())
})
t.Run("test column field data", func(t *testing.T) {
fd := column.FieldData()
assert.NotNil(t, fd)
assert.Equal(t, fd.GetFieldName(), columnName)
})
t.Run("test column value by idx", func(t *testing.T) {
_, err := column.ValueByIdx(-1)
assert.NotNil(t, err)
_, err = column.ValueByIdx(columnLen)
assert.NotNil(t, err)
for i := 0; i < columnLen; i++ {
v, err := column.ValueByIdx(i)
assert.Nil(t, err)
assert.Equal(t, column.values[i], v)
}
})
}
func TestFieldDataVarCharColumn(t *testing.T) {
colLen := rand.Intn(10) + 8
name := fmt.Sprintf("fd_VarChar_%d", rand.Int())
fd := &schemapb.FieldData{
Type: schemapb.DataType_VarChar,
FieldName: name,
}
t.Run("normal usage", func(t *testing.T) {
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_StringData{
StringData: &schemapb.StringArray{
Data: make([]string, colLen),
},
},
},
}
column, err := FieldDataColumn(fd, 0, colLen)
assert.Nil(t, err)
assert.NotNil(t, column)
assert.Equal(t, name, column.Name())
assert.Equal(t, colLen, column.Len())
assert.Equal(t, entity.FieldTypeVarChar, column.Type())
var ev string
err = column.AppendValue(ev)
assert.Equal(t, colLen+1, column.Len())
assert.Nil(t, err)
err = column.AppendValue(struct{}{})
assert.Equal(t, colLen+1, column.Len())
assert.NotNil(t, err)
})
t.Run("nil data", func(t *testing.T) {
fd.Field = nil
_, err := FieldDataColumn(fd, 0, colLen)
assert.NotNil(t, err)
})
t.Run("get all data", func(t *testing.T) {
fd.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_StringData{
StringData: &schemapb.StringArray{
Data: make([]string, colLen),
},
},
},
}
column, err := FieldDataColumn(fd, 0, -1)
assert.Nil(t, err)
assert.NotNil(t, column)
assert.Equal(t, name, column.Name())
assert.Equal(t, colLen, column.Len())
assert.Equal(t, entity.FieldTypeVarChar, column.Type())
})
}

158
client/column/vector.go Normal file
View File

@ -0,0 +1,158 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package column
import (
"github.com/cockroachdb/errors"
"github.com/samber/lo"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/client/v2/entity"
)
type vectorBase[T entity.Vector] struct {
*genericColumnBase[T]
dim int
}
func (b *vectorBase[T]) Dim() int {
return b.dim
}
func (b *vectorBase[T]) FieldData() *schemapb.FieldData {
fd := b.genericColumnBase.FieldData()
vectors := fd.GetVectors()
vectors.Dim = int64(b.dim)
return fd
}
func newVectorBase[T entity.Vector](fieldName string, dim int, vectors []T, fieldType entity.FieldType) *vectorBase[T] {
return &vectorBase[T]{
genericColumnBase: &genericColumnBase[T]{
name: fieldName,
fieldType: fieldType,
values: vectors,
},
dim: dim,
}
}
/* float vector */
type ColumnFloatVector struct {
*vectorBase[entity.FloatVector]
}
func NewColumnFloatVector(fieldName string, dim int, data [][]float32) *ColumnFloatVector {
vectors := lo.Map(data, func(row []float32, _ int) entity.FloatVector { return entity.FloatVector(row) })
return &ColumnFloatVector{
vectorBase: newVectorBase(fieldName, dim, vectors, entity.FieldTypeFloatVector),
}
}
// AppendValue appends vector value into values.
// override default type constrains, add `[]float32` conversion
func (c *ColumnFloatVector) AppendValue(i interface{}) error {
switch vector := i.(type) {
case entity.FloatVector:
c.values = append(c.values, vector)
case []float32:
c.values = append(c.values, vector)
default:
return errors.Newf("unexpected append value type %T, field type %v", vector, c.fieldType)
}
return nil
}
/* binary vector */
type ColumnBinaryVector struct {
*vectorBase[entity.BinaryVector]
}
func NewColumnBinaryVector(fieldName string, dim int, data [][]byte) *ColumnBinaryVector {
vectors := lo.Map(data, func(row []byte, _ int) entity.BinaryVector { return entity.BinaryVector(row) })
return &ColumnBinaryVector{
vectorBase: newVectorBase(fieldName, dim, vectors, entity.FieldTypeBinaryVector),
}
}
// AppendValue appends vector value into values.
// override default type constrains, add `[]byte` conversion
func (c *ColumnBinaryVector) AppendValue(i interface{}) error {
switch vector := i.(type) {
case entity.BinaryVector:
c.values = append(c.values, vector)
case []byte:
c.values = append(c.values, vector)
default:
return errors.Newf("unexpected append value type %T, field type %v", vector, c.fieldType)
}
return nil
}
/* fp16 vector */
type ColumnFloat16Vector struct {
*vectorBase[entity.Float16Vector]
}
func NewColumnFloat16Vector(fieldName string, dim int, data [][]byte) *ColumnFloat16Vector {
vectors := lo.Map(data, func(row []byte, _ int) entity.Float16Vector { return entity.Float16Vector(row) })
return &ColumnFloat16Vector{
vectorBase: newVectorBase(fieldName, dim, vectors, entity.FieldTypeFloat16Vector),
}
}
// AppendValue appends vector value into values.
// override default type constrains, add `[]byte` conversion
func (c *ColumnFloat16Vector) AppendValue(i interface{}) error {
switch vector := i.(type) {
case entity.Float16Vector:
c.values = append(c.values, vector)
case []byte:
c.values = append(c.values, vector)
default:
return errors.Newf("unexpected append value type %T, field type %v", vector, c.fieldType)
}
return nil
}
type ColumnBFloat16Vector struct {
*vectorBase[entity.BFloat16Vector]
}
func NewColumnBFloat16Vector(fieldName string, dim int, data [][]byte) *ColumnBFloat16Vector {
vectors := lo.Map(data, func(row []byte, _ int) entity.BFloat16Vector { return entity.BFloat16Vector(row) })
return &ColumnBFloat16Vector{
vectorBase: newVectorBase(fieldName, dim, vectors, entity.FieldTypeBFloat16Vector),
}
}
// AppendValue appends vector value into values.
// override default type constrains, add `[]byte` conversion
func (c *ColumnBFloat16Vector) AppendValue(i interface{}) error {
switch vector := i.(type) {
case entity.BFloat16Vector:
c.values = append(c.values, vector)
case []byte:
c.values = append(c.values, vector)
default:
return errors.Newf("unexpected append value type %T, field type %v", vector, c.fieldType)
}
return nil
}

434
client/column/vector_gen.go
View File

@ -1,434 +0,0 @@
// Code generated by go generate; DO NOT EDIT
// This file is generated by go generated
package column
import (
"fmt"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/cockroachdb/errors"
)
// ColumnBinaryVector generated columns type for BinaryVector
type ColumnBinaryVector struct {
ColumnBase
name string
dim int
values [][]byte
}
// Name returns column name
func (c *ColumnBinaryVector) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnBinaryVector) Type() entity.FieldType {
return entity.FieldTypeBinaryVector
}
// Len returns column data length
func (c *ColumnBinaryVector) Len() int {
return len(c.values)
}
func (c *ColumnBinaryVector) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
if end == -1 || end > l {
end = l
}
return &ColumnBinaryVector{
ColumnBase: c.ColumnBase,
name: c.name,
dim: c.dim,
values: c.values[start:end],
}
}
// Dim returns vector dimension
func (c *ColumnBinaryVector) Dim() int {
return c.dim
}
// Get returns values at index as interface{}.
func (c *ColumnBinaryVector) Get(idx int) (interface{}, error) {
if idx < 0 || idx >= c.Len() {
return nil, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnBinaryVector) AppendValue(i interface{}) error {
v, ok := i.([]byte)
if !ok {
return fmt.Errorf("invalid type, expected []byte, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnBinaryVector) Data() [][]byte {
return c.values
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnBinaryVector) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_BinaryVector,
FieldName: c.name,
}
data := make([]byte, 0, len(c.values)*c.dim)
for _, vector := range c.values {
data = append(data, vector...)
}
fd.Field = &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: int64(c.dim),
Data: &schemapb.VectorField_BinaryVector{
BinaryVector: data,
},
},
}
return fd
}
// NewColumnBinaryVector auto generated constructor
func NewColumnBinaryVector(name string, dim int, values [][]byte) *ColumnBinaryVector {
return &ColumnBinaryVector{
name: name,
dim: dim,
values: values,
}
}
// ColumnFloatVector generated columns type for FloatVector
type ColumnFloatVector struct {
ColumnBase
name string
dim int
values [][]float32
}
// Name returns column name
func (c *ColumnFloatVector) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnFloatVector) Type() entity.FieldType {
return entity.FieldTypeFloatVector
}
// Len returns column data length
func (c *ColumnFloatVector) Len() int {
return len(c.values)
}
func (c *ColumnFloatVector) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
if end == -1 || end > l {
end = l
}
return &ColumnFloatVector{
ColumnBase: c.ColumnBase,
name: c.name,
dim: c.dim,
values: c.values[start:end],
}
}
// Dim returns vector dimension
func (c *ColumnFloatVector) Dim() int {
return c.dim
}
// Get returns values at index as interface{}.
func (c *ColumnFloatVector) Get(idx int) (interface{}, error) {
if idx < 0 || idx >= c.Len() {
return nil, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnFloatVector) AppendValue(i interface{}) error {
v, ok := i.([]float32)
if !ok {
return fmt.Errorf("invalid type, expected []float32, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnFloatVector) Data() [][]float32 {
return c.values
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnFloatVector) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_FloatVector,
FieldName: c.name,
}
data := make([]float32, 0, len(c.values)*c.dim)
for _, vector := range c.values {
data = append(data, vector...)
}
fd.Field = &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: int64(c.dim),
Data: &schemapb.VectorField_FloatVector{
FloatVector: &schemapb.FloatArray{
Data: data,
},
},
},
}
return fd
}
// NewColumnFloatVector auto generated constructor
func NewColumnFloatVector(name string, dim int, values [][]float32) *ColumnFloatVector {
return &ColumnFloatVector{
name: name,
dim: dim,
values: values,
}
}
// ColumnFloat16Vector generated columns type for Float16Vector
type ColumnFloat16Vector struct {
ColumnBase
name string
dim int
values [][]byte
}
// Name returns column name
func (c *ColumnFloat16Vector) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnFloat16Vector) Type() entity.FieldType {
return entity.FieldTypeFloat16Vector
}
// Len returns column data length
func (c *ColumnFloat16Vector) Len() int {
return len(c.values)
}
func (c *ColumnFloat16Vector) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
if end == -1 || end > l {
end = l
}
return &ColumnFloat16Vector{
ColumnBase: c.ColumnBase,
name: c.name,
dim: c.dim,
values: c.values[start:end],
}
}
// Dim returns vector dimension
func (c *ColumnFloat16Vector) Dim() int {
return c.dim
}
// Get returns values at index as interface{}.
func (c *ColumnFloat16Vector) Get(idx int) (interface{}, error) {
if idx < 0 || idx >= c.Len() {
return nil, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnFloat16Vector) AppendValue(i interface{}) error {
v, ok := i.([]byte)
if !ok {
return fmt.Errorf("invalid type, expected []byte, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnFloat16Vector) Data() [][]byte {
return c.values
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnFloat16Vector) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_Float16Vector,
FieldName: c.name,
}
data := make([]byte, 0, len(c.values)*c.dim*2)
for _, vector := range c.values {
data = append(data, vector...)
}
fd.Field = &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: int64(c.dim),
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: data,
},
},
}
return fd
}
// NewColumnFloat16Vector auto generated constructor
func NewColumnFloat16Vector(name string, dim int, values [][]byte) *ColumnFloat16Vector {
return &ColumnFloat16Vector{
name: name,
dim: dim,
values: values,
}
}
// ColumnBFloat16Vector generated columns type for BFloat16Vector
type ColumnBFloat16Vector struct {
ColumnBase
name string
dim int
values [][]byte
}
// Name returns column name
func (c *ColumnBFloat16Vector) Name() string {
return c.name
}
// Type returns column entity.FieldType
func (c *ColumnBFloat16Vector) Type() entity.FieldType {
return entity.FieldTypeBFloat16Vector
}
// Len returns column data length
func (c *ColumnBFloat16Vector) Len() int {
return len(c.values)
}
func (c *ColumnBFloat16Vector) Slice(start, end int) Column {
l := c.Len()
if start > l {
start = l
}
if end == -1 || end > l {
end = l
}
if end == -1 || end > l {
end = l
}
return &ColumnBFloat16Vector{
ColumnBase: c.ColumnBase,
name: c.name,
dim: c.dim,
values: c.values[start:end],
}
}
// Dim returns vector dimension
func (c *ColumnBFloat16Vector) Dim() int {
return c.dim
}
// Get returns values at index as interface{}.
func (c *ColumnBFloat16Vector) Get(idx int) (interface{}, error) {
if idx < 0 || idx >= c.Len() {
return nil, errors.New("index out of range")
}
return c.values[idx], nil
}
// AppendValue append value into column
func (c *ColumnBFloat16Vector) AppendValue(i interface{}) error {
v, ok := i.([]byte)
if !ok {
return fmt.Errorf("invalid type, expected []byte, got %T", i)
}
c.values = append(c.values, v)
return nil
}
// Data returns column data
func (c *ColumnBFloat16Vector) Data() [][]byte {
return c.values
}
// FieldData return column data mapped to schemapb.FieldData
func (c *ColumnBFloat16Vector) FieldData() *schemapb.FieldData {
fd := &schemapb.FieldData{
Type: schemapb.DataType_BFloat16Vector,
FieldName: c.name,
}
data := make([]byte, 0, len(c.values)*c.dim*2)
for _, vector := range c.values {
data = append(data, vector...)
}
fd.Field = &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: int64(c.dim),
Data: &schemapb.VectorField_Bfloat16Vector{
Bfloat16Vector: data,
},
},
}
return fd
}
// NewColumnBFloat16Vector auto generated constructor
func NewColumnBFloat16Vector(name string, dim int, values [][]byte) *ColumnBFloat16Vector {
return &ColumnBFloat16Vector{
name: name,
dim: dim,
values: values,
}
}

264
client/column/vector_gen_test.go
View File

@ -1,264 +0,0 @@
// Code generated by go generate; DO NOT EDIT
// This file is generated by go generated
package column
import (
"fmt"
"math/rand"
"testing"
"time"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/stretchr/testify/assert"
)
func TestColumnBinaryVector(t *testing.T) {
rand.Seed(time.Now().UnixNano())
columnName := fmt.Sprintf("column_BinaryVector_%d", rand.Int())
columnLen := 12 + rand.Intn(10)
dim := ([]int{64, 128, 256, 512})[rand.Intn(4)]
v := make([][]byte, 0, columnLen)
dlen := dim
dlen /= 8
for i := 0; i < columnLen; i++ {
entry := make([]byte, dlen)
v = append(v, entry)
}
column := NewColumnBinaryVector(columnName, dim, v)
t.Run("test meta", func(t *testing.T) {
ft := entity.FieldTypeBinaryVector
assert.Equal(t, "BinaryVector", ft.Name())
assert.Equal(t, "[]byte", ft.String())
pbName, pbType := ft.PbFieldType()
assert.Equal(t, "[]byte", pbName)
assert.Equal(t, "", pbType)
})
t.Run("test column attribute", func(t *testing.T) {
assert.Equal(t, columnName, column.Name())
assert.Equal(t, entity.FieldTypeBinaryVector, column.Type())
assert.Equal(t, columnLen, column.Len())
assert.Equal(t, dim, column.Dim())
assert.Equal(t, v, column.Data())
var ev []byte
err := column.AppendValue(ev)
assert.Equal(t, columnLen+1, column.Len())
assert.Nil(t, err)
err = column.AppendValue(struct{}{})
assert.Equal(t, columnLen+1, column.Len())
assert.NotNil(t, err)
})
t.Run("test column field data", func(t *testing.T) {
fd := column.FieldData()
assert.NotNil(t, fd)
assert.Equal(t, fd.GetFieldName(), columnName)
c, err := FieldDataVector(fd)
assert.NotNil(t, c)
assert.NoError(t, err)
})
t.Run("test column field data error", func(t *testing.T) {
fd := &schemapb.FieldData{
Type: schemapb.DataType_BinaryVector,
FieldName: columnName,
}
_, err := FieldDataVector(fd)
assert.Error(t, err)
})
}
func TestColumnFloatVector(t *testing.T) {
rand.Seed(time.Now().UnixNano())
columnName := fmt.Sprintf("column_FloatVector_%d", rand.Int())
columnLen := 12 + rand.Intn(10)
dim := ([]int{64, 128, 256, 512})[rand.Intn(4)]
v := make([][]float32, 0, columnLen)
dlen := dim
for i := 0; i < columnLen; i++ {
entry := make([]float32, dlen)
v = append(v, entry)
}
column := NewColumnFloatVector(columnName, dim, v)
t.Run("test meta", func(t *testing.T) {
ft := entity.FieldTypeFloatVector
assert.Equal(t, "FloatVector", ft.Name())
assert.Equal(t, "[]float32", ft.String())
pbName, pbType := ft.PbFieldType()
assert.Equal(t, "[]float32", pbName)
assert.Equal(t, "", pbType)
})
t.Run("test column attribute", func(t *testing.T) {
assert.Equal(t, columnName, column.Name())
assert.Equal(t, entity.FieldTypeFloatVector, column.Type())
assert.Equal(t, columnLen, column.Len())
assert.Equal(t, dim, column.Dim())
assert.Equal(t, v, column.Data())
var ev []float32
err := column.AppendValue(ev)
assert.Equal(t, columnLen+1, column.Len())
assert.Nil(t, err)
err = column.AppendValue(struct{}{})
assert.Equal(t, columnLen+1, column.Len())
assert.NotNil(t, err)
})
t.Run("test column field data", func(t *testing.T) {
fd := column.FieldData()
assert.NotNil(t, fd)
assert.Equal(t, fd.GetFieldName(), columnName)
c, err := FieldDataVector(fd)
assert.NotNil(t, c)
assert.NoError(t, err)
})
t.Run("test column field data error", func(t *testing.T) {
fd := &schemapb.FieldData{
Type: schemapb.DataType_FloatVector,
FieldName: columnName,
}
_, err := FieldDataVector(fd)
assert.Error(t, err)
})
}
func TestColumnFloat16Vector(t *testing.T) {
rand.Seed(time.Now().UnixNano())
columnName := fmt.Sprintf("column_Float16Vector_%d", rand.Int())
columnLen := 12 + rand.Intn(10)
dim := ([]int{64, 128, 256, 512})[rand.Intn(4)]
v := make([][]byte, 0, columnLen)
dlen := dim
dlen *= 2
for i := 0; i < columnLen; i++ {
entry := make([]byte, dlen)
v = append(v, entry)
}
column := NewColumnFloat16Vector(columnName, dim, v)
t.Run("test meta", func(t *testing.T) {
ft := entity.FieldTypeFloat16Vector
assert.Equal(t, "Float16Vector", ft.Name())
assert.Equal(t, "[]byte", ft.String())
pbName, pbType := ft.PbFieldType()
assert.Equal(t, "[]byte", pbName)
assert.Equal(t, "", pbType)
})
t.Run("test column attribute", func(t *testing.T) {
assert.Equal(t, columnName, column.Name())
assert.Equal(t, entity.FieldTypeFloat16Vector, column.Type())
assert.Equal(t, columnLen, column.Len())
assert.Equal(t, dim, column.Dim())
assert.Equal(t, v, column.Data())
var ev []byte
err := column.AppendValue(ev)
assert.Equal(t, columnLen+1, column.Len())
assert.Nil(t, err)
err = column.AppendValue(struct{}{})
assert.Equal(t, columnLen+1, column.Len())
assert.NotNil(t, err)
})
t.Run("test column field data", func(t *testing.T) {
fd := column.FieldData()
assert.NotNil(t, fd)
assert.Equal(t, fd.GetFieldName(), columnName)
c, err := FieldDataVector(fd)
assert.NotNil(t, c)
assert.NoError(t, err)
})
t.Run("test column field data error", func(t *testing.T) {
fd := &schemapb.FieldData{
Type: schemapb.DataType_Float16Vector,
FieldName: columnName,
}
_, err := FieldDataVector(fd)
assert.Error(t, err)
})
}
func TestColumnBFloat16Vector(t *testing.T) {
rand.Seed(time.Now().UnixNano())
columnName := fmt.Sprintf("column_BFloat16Vector_%d", rand.Int())
columnLen := 12 + rand.Intn(10)
dim := ([]int{64, 128, 256, 512})[rand.Intn(4)]
v := make([][]byte, 0, columnLen)
dlen := dim
dlen *= 2
for i := 0; i < columnLen; i++ {
entry := make([]byte, dlen)
v = append(v, entry)
}
column := NewColumnBFloat16Vector(columnName, dim, v)
t.Run("test meta", func(t *testing.T) {
ft := entity.FieldTypeBFloat16Vector
assert.Equal(t, "BFloat16Vector", ft.Name())
assert.Equal(t, "[]byte", ft.String())
pbName, pbType := ft.PbFieldType()
assert.Equal(t, "[]byte", pbName)
assert.Equal(t, "", pbType)
})
t.Run("test column attribute", func(t *testing.T) {
assert.Equal(t, columnName, column.Name())
assert.Equal(t, entity.FieldTypeBFloat16Vector, column.Type())
assert.Equal(t, columnLen, column.Len())
assert.Equal(t, dim, column.Dim())
assert.Equal(t, v, column.Data())
var ev []byte
err := column.AppendValue(ev)
assert.Equal(t, columnLen+1, column.Len())
assert.Nil(t, err)
err = column.AppendValue(struct{}{})
assert.Equal(t, columnLen+1, column.Len())
assert.NotNil(t, err)
})
t.Run("test column field data", func(t *testing.T) {
fd := column.FieldData()
assert.NotNil(t, fd)
assert.Equal(t, fd.GetFieldName(), columnName)
c, err := FieldDataVector(fd)
assert.NotNil(t, c)
assert.NoError(t, err)
})
t.Run("test column field data error", func(t *testing.T) {
fd := &schemapb.FieldData{
Type: schemapb.DataType_BFloat16Vector,
FieldName: columnName,
}
_, err := FieldDataVector(fd)
assert.Error(t, err)
})
}

167
client/column/vector_test.go Normal file
View File

@ -0,0 +1,167 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package column
import (
"fmt"
"math"
"math/rand"
"testing"
"github.com/samber/lo"
"github.com/stretchr/testify/suite"
"github.com/milvus-io/milvus/client/v2/entity"
)
type VectorSuite struct {
suite.Suite
}
func (s *VectorSuite) TestBasic() {
s.Run("float_vector", func() {
name := fmt.Sprintf("field_%d", rand.Intn(1000))
n := 3
dim := rand.Intn(10) + 2
data := make([][]float32, 0, n)
for i := 0; i < n; i++ {
row := lo.RepeatBy(dim, func(i int) float32 {
return rand.Float32()
})
data = append(data, row)
}
column := NewColumnFloatVector(name, dim, data)
s.Equal(entity.FieldTypeFloatVector, column.Type())
s.Equal(name, column.Name())
s.Equal(lo.Map(data, func(row []float32, _ int) entity.FloatVector { return entity.FloatVector(row) }), column.Data())
s.Equal(dim, column.Dim())
fd := column.FieldData()
s.Equal(name, fd.GetFieldName())
s.Equal(lo.Flatten(data), fd.GetVectors().GetFloatVector().GetData())
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnFloatVector)
if s.True(ok) {
s.Equal(entity.FieldTypeFloatVector, parsed.Type())
s.Equal(name, parsed.Name())
s.Equal(lo.Map(data, func(row []float32, _ int) entity.FloatVector { return entity.FloatVector(row) }), parsed.Data())
s.Equal(dim, parsed.Dim())
}
})
s.Run("binary_vector", func() {
name := fmt.Sprintf("field_%d", rand.Intn(1000))
n := 3
dim := (rand.Intn(10) + 1) * 8
data := make([][]byte, 0, n)
for i := 0; i < n; i++ {
row := lo.RepeatBy(dim/8, func(i int) byte {
return byte(rand.Intn(math.MaxUint8))
})
data = append(data, row)
}
column := NewColumnBinaryVector(name, dim, data)
s.Equal(entity.FieldTypeBinaryVector, column.Type())
s.Equal(name, column.Name())
s.Equal(lo.Map(data, func(row []byte, _ int) entity.BinaryVector { return entity.BinaryVector(row) }), column.Data())
s.Equal(dim, column.Dim())
fd := column.FieldData()
s.Equal(name, fd.GetFieldName())
s.Equal(lo.Flatten(data), fd.GetVectors().GetBinaryVector())
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnBinaryVector)
if s.True(ok) {
s.Equal(entity.FieldTypeBinaryVector, parsed.Type())
s.Equal(name, parsed.Name())
s.Equal(lo.Map(data, func(row []byte, _ int) entity.BinaryVector { return entity.BinaryVector(row) }), parsed.Data())
s.Equal(dim, parsed.Dim())
}
})
s.Run("fp16_vector", func() {
name := fmt.Sprintf("field_%d", rand.Intn(1000))
n := 3
dim := rand.Intn(10) + 1
data := make([][]byte, 0, n)
for i := 0; i < n; i++ {
row := lo.RepeatBy(dim*2, func(i int) byte {
return byte(rand.Intn(math.MaxUint8))
})
data = append(data, row)
}
column := NewColumnFloat16Vector(name, dim, data)
s.Equal(entity.FieldTypeFloat16Vector, column.Type())
s.Equal(name, column.Name())
s.Equal(lo.Map(data, func(row []byte, _ int) entity.Float16Vector { return entity.Float16Vector(row) }), column.Data())
s.Equal(dim, column.Dim())
fd := column.FieldData()
s.Equal(name, fd.GetFieldName())
s.Equal(lo.Flatten(data), fd.GetVectors().GetFloat16Vector())
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnFloat16Vector)
if s.True(ok) {
s.Equal(entity.FieldTypeFloat16Vector, parsed.Type())
s.Equal(name, parsed.Name())
s.Equal(lo.Map(data, func(row []byte, _ int) entity.Float16Vector { return entity.Float16Vector(row) }), parsed.Data())
s.Equal(dim, parsed.Dim())
}
})
s.Run("bf16_vector", func() {
name := fmt.Sprintf("field_%d", rand.Intn(1000))
n := 3
dim := rand.Intn(10) + 1
data := make([][]byte, 0, n)
for i := 0; i < n; i++ {
row := lo.RepeatBy(dim*2, func(i int) byte {
return byte(rand.Intn(math.MaxUint8))
})
data = append(data, row)
}
column := NewColumnBFloat16Vector(name, dim, data)
s.Equal(entity.FieldTypeBFloat16Vector, column.Type())
s.Equal(name, column.Name())
s.Equal(lo.Map(data, func(row []byte, _ int) entity.BFloat16Vector { return entity.BFloat16Vector(row) }), column.Data())
s.Equal(dim, column.Dim())
fd := column.FieldData()
s.Equal(name, fd.GetFieldName())
s.Equal(lo.Flatten(data), fd.GetVectors().GetBfloat16Vector())
result, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
parsed, ok := result.(*ColumnBFloat16Vector)
if s.True(ok) {
s.Equal(entity.FieldTypeBFloat16Vector, parsed.Type())
s.Equal(name, parsed.Name())
s.Equal(lo.Map(data, func(row []byte, _ int) entity.BFloat16Vector { return entity.BFloat16Vector(row) }), parsed.Data())
s.Equal(dim, parsed.Dim())
}
})
}
func TestVectors(t *testing.T) {
suite.Run(t, new(VectorSuite))
}