mirror of https://github.com/milvus-io/milvus.git
299 lines
8.7 KiB
Go
299 lines
8.7 KiB
Go
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package accesslog
|
|
|
|
import (
|
|
"bytes"
|
|
"io"
|
|
"os"
|
|
"path"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
"github.com/milvus-io/milvus/pkg/util/paramtable"
|
|
)
|
|
|
|
func getText(size int) []byte {
|
|
text := make([]byte, size)
|
|
|
|
for i := 0; i < size; i++ {
|
|
text[i] = byte('-')
|
|
}
|
|
return text
|
|
}
|
|
|
|
func TestRotateWriter_Basic(t *testing.T) {
|
|
var Params paramtable.ComponentParam
|
|
Params.Init(paramtable.NewBaseTable(paramtable.SkipRemote(true)))
|
|
testPath := "/tmp/accesstest"
|
|
Params.Save(Params.ProxyCfg.AccessLog.Enable.Key, "true")
|
|
Params.Save(Params.ProxyCfg.AccessLog.Filename.Key, "test_access")
|
|
Params.Save(Params.ProxyCfg.AccessLog.LocalPath.Key, testPath)
|
|
Params.Save(Params.ProxyCfg.AccessLog.MinioEnable.Key, "true")
|
|
Params.Save(Params.ProxyCfg.AccessLog.RemotePath.Key, "access_log/")
|
|
defer os.RemoveAll(testPath)
|
|
|
|
logger, err := NewRotateWriter(&Params.ProxyCfg.AccessLog, &Params.MinioCfg)
|
|
assert.NoError(t, err)
|
|
defer logger.handler.Clean()
|
|
defer logger.Close()
|
|
|
|
num := 100
|
|
text := getText(num)
|
|
n, err := logger.Write(text)
|
|
assert.Equal(t, num, n)
|
|
assert.NoError(t, err)
|
|
|
|
err = logger.Rotate()
|
|
assert.NoError(t, err)
|
|
|
|
time.Sleep(time.Duration(1) * time.Second)
|
|
logfiles, err := logger.handler.listAll()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 1, len(logfiles))
|
|
}
|
|
|
|
func TestRotateWriter_TimeRotate(t *testing.T) {
|
|
var Params paramtable.ComponentParam
|
|
Params.Init(paramtable.NewBaseTable(paramtable.SkipRemote(true)))
|
|
testPath := "/tmp/accesstest"
|
|
Params.Save(Params.ProxyCfg.AccessLog.Enable.Key, "true")
|
|
Params.Save(Params.ProxyCfg.AccessLog.Filename.Key, "test_access")
|
|
Params.Save(Params.ProxyCfg.AccessLog.LocalPath.Key, testPath)
|
|
Params.Save(Params.ProxyCfg.AccessLog.MinioEnable.Key, "true")
|
|
Params.Save(Params.ProxyCfg.AccessLog.RemotePath.Key, "access_log/")
|
|
Params.Save(Params.ProxyCfg.AccessLog.RotatedTime.Key, "2")
|
|
Params.Save(Params.ProxyCfg.AccessLog.MaxBackups.Key, "0")
|
|
defer os.RemoveAll(testPath)
|
|
|
|
logger, err := NewRotateWriter(&Params.ProxyCfg.AccessLog, &Params.MinioCfg)
|
|
assert.NoError(t, err)
|
|
defer logger.handler.Clean()
|
|
defer logger.Close()
|
|
|
|
num := 100
|
|
text := getText(num)
|
|
n, err := logger.Write(text)
|
|
assert.Equal(t, num, n)
|
|
assert.NoError(t, err)
|
|
|
|
time.Sleep(time.Duration(4) * time.Second)
|
|
logfiles, err := logger.handler.listAll()
|
|
assert.NoError(t, err)
|
|
assert.GreaterOrEqual(t, len(logfiles), 1)
|
|
}
|
|
|
|
func TestRotateWriter_SizeRotate(t *testing.T) {
|
|
var Params paramtable.ComponentParam
|
|
Params.Init(paramtable.NewBaseTable(paramtable.SkipRemote(true)))
|
|
testPath := "/tmp/accesstest"
|
|
Params.Save(Params.ProxyCfg.AccessLog.Enable.Key, "true")
|
|
Params.Save(Params.ProxyCfg.AccessLog.Filename.Key, "test_access")
|
|
Params.Save(Params.ProxyCfg.AccessLog.LocalPath.Key, testPath)
|
|
Params.Save(Params.ProxyCfg.AccessLog.MinioEnable.Key, "true")
|
|
Params.Save(Params.ProxyCfg.AccessLog.RemotePath.Key, "access_log/")
|
|
Params.Save(Params.ProxyCfg.AccessLog.MaxSize.Key, "1")
|
|
defer os.RemoveAll(testPath)
|
|
|
|
logger, err := NewRotateWriter(&Params.ProxyCfg.AccessLog, &Params.MinioCfg)
|
|
assert.NoError(t, err)
|
|
defer logger.handler.Clean()
|
|
defer logger.Close()
|
|
|
|
num := 1024 * 1024
|
|
text := getText(num + 1)
|
|
_, err = logger.Write(text)
|
|
assert.Error(t, err)
|
|
|
|
for i := 1; i <= 2; i++ {
|
|
text = getText(num)
|
|
n, err := logger.Write(text)
|
|
assert.Equal(t, num, n)
|
|
assert.NoError(t, err)
|
|
}
|
|
|
|
time.Sleep(time.Duration(1) * time.Second)
|
|
logfiles, err := logger.handler.listAll()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 1, len(logfiles))
|
|
}
|
|
|
|
func TestRotateWriter_LocalRetention(t *testing.T) {
|
|
var Params paramtable.ComponentParam
|
|
Params.Init(paramtable.NewBaseTable(paramtable.SkipRemote(true)))
|
|
testPath := "/tmp/accesstest"
|
|
Params.Save(Params.ProxyCfg.AccessLog.Enable.Key, "true")
|
|
Params.Save(Params.ProxyCfg.AccessLog.Filename.Key, "test_access")
|
|
Params.Save(Params.ProxyCfg.AccessLog.LocalPath.Key, testPath)
|
|
Params.Save(Params.ProxyCfg.AccessLog.MaxBackups.Key, "1")
|
|
defer os.RemoveAll(testPath)
|
|
|
|
logger, err := NewRotateWriter(&Params.ProxyCfg.AccessLog, &Params.MinioCfg)
|
|
assert.NoError(t, err)
|
|
defer logger.Close()
|
|
|
|
logger.Rotate()
|
|
logger.Rotate()
|
|
time.Sleep(time.Duration(1) * time.Second)
|
|
logFiles, err := logger.oldLogFiles()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 1, len(logFiles))
|
|
}
|
|
|
|
func TestRotateWriter_BasicError(t *testing.T) {
|
|
var Params paramtable.ComponentParam
|
|
Params.Init(paramtable.NewBaseTable(paramtable.SkipRemote(true)))
|
|
testPath := ""
|
|
Params.Save(Params.ProxyCfg.AccessLog.Enable.Key, "true")
|
|
Params.Save(Params.ProxyCfg.AccessLog.Filename.Key, "test_access")
|
|
Params.Save(Params.ProxyCfg.AccessLog.LocalPath.Key, testPath)
|
|
|
|
logger, err := NewRotateWriter(&Params.ProxyCfg.AccessLog, &Params.MinioCfg)
|
|
assert.NoError(t, err)
|
|
defer os.RemoveAll(logger.dir())
|
|
defer logger.Close()
|
|
|
|
logger.openFileExistingOrNew()
|
|
|
|
os.Mkdir(path.Join(logger.dir(), "test"), 0o744)
|
|
logfile, err := logger.oldLogFiles()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 0, len(logfile))
|
|
|
|
_, err = timeFromName("a.b", "a", "c")
|
|
assert.Error(t, err)
|
|
_, err = timeFromName("a.b", "d", "c")
|
|
assert.Error(t, err)
|
|
}
|
|
|
|
func TestRotateWriter_InitError(t *testing.T) {
|
|
var params paramtable.ComponentParam
|
|
params.Init(paramtable.NewBaseTable(paramtable.SkipRemote(true)))
|
|
testPath := "/tmp/test"
|
|
params.Save(params.ProxyCfg.AccessLog.Enable.Key, "true")
|
|
params.Save(params.ProxyCfg.AccessLog.Filename.Key, "test_access")
|
|
params.Save(params.ProxyCfg.AccessLog.LocalPath.Key, testPath)
|
|
params.Save(params.ProxyCfg.AccessLog.MinioEnable.Key, "true")
|
|
params.Save(params.MinioCfg.Address.Key, "")
|
|
// init err with invalid minio address
|
|
_, err := NewRotateWriter(¶ms.ProxyCfg.AccessLog, ¶ms.MinioCfg)
|
|
assert.Error(t, err)
|
|
}
|
|
|
|
func TestRotateWriter_Close(t *testing.T) {
|
|
var Params paramtable.ComponentParam
|
|
|
|
Params.Init(paramtable.NewBaseTable(paramtable.SkipRemote(true)))
|
|
testPath := "/tmp/accesstest"
|
|
Params.Save(Params.ProxyCfg.AccessLog.Enable.Key, "true")
|
|
Params.Save(Params.ProxyCfg.AccessLog.Filename.Key, "test_access")
|
|
Params.Save(Params.ProxyCfg.AccessLog.LocalPath.Key, testPath)
|
|
Params.Save(Params.ProxyCfg.AccessLog.CacheSize.Key, "0")
|
|
|
|
logger, err := NewRotateWriter(&Params.ProxyCfg.AccessLog, &Params.MinioCfg)
|
|
assert.NoError(t, err)
|
|
defer os.RemoveAll(logger.dir())
|
|
|
|
_, err = logger.Write([]byte("test"))
|
|
assert.NoError(t, err)
|
|
|
|
logger.Close()
|
|
|
|
_, err = logger.Write([]byte("test"))
|
|
assert.Error(t, err)
|
|
}
|
|
|
|
func TestCacheWriter_Normal(t *testing.T) {
|
|
buffer := bytes.NewBuffer(make([]byte, 0))
|
|
writer := NewCacheWriter(buffer, 512, 0)
|
|
|
|
writer.Write([]byte("111\n"))
|
|
_, err := buffer.ReadByte()
|
|
assert.Error(t, err, io.EOF)
|
|
|
|
writer.Flush()
|
|
b, err := buffer.ReadBytes('\n')
|
|
assert.Equal(t, 4, len(b))
|
|
assert.NoError(t, err)
|
|
|
|
writer.Write([]byte(strings.Repeat("1", 512) + "\n"))
|
|
b, err = buffer.ReadBytes('\n')
|
|
assert.Equal(t, 513, len(b))
|
|
assert.NoError(t, err)
|
|
|
|
writer.Close()
|
|
// writer to closed writer
|
|
_, err = writer.Write([]byte(strings.Repeat("1", 512) + "\n"))
|
|
assert.Error(t, err)
|
|
}
|
|
|
|
type TestWriter struct {
|
|
closed bool
|
|
buffer *bytes.Buffer
|
|
mu sync.Mutex
|
|
}
|
|
|
|
func (w *TestWriter) Write(p []byte) (n int, err error) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
|
|
return w.buffer.Write(p)
|
|
}
|
|
|
|
func (w *TestWriter) ReadBytes(delim byte) (line []byte, err error) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
|
|
return w.buffer.ReadBytes(delim)
|
|
}
|
|
|
|
func (w *TestWriter) ReadByte() (byte, error) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
|
|
return w.buffer.ReadByte()
|
|
}
|
|
|
|
func (w *TestWriter) Close() error {
|
|
w.closed = true
|
|
return nil
|
|
}
|
|
|
|
func TestCacheWriter_WithAutoFlush(t *testing.T) {
|
|
buffer := &TestWriter{buffer: bytes.NewBuffer(make([]byte, 0))}
|
|
writer := NewCacheWriterWithCloser(buffer, buffer, 512, 1*time.Second)
|
|
writer.Write([]byte("111\n"))
|
|
_, err := buffer.ReadByte()
|
|
assert.Error(t, err, io.EOF)
|
|
|
|
assert.Eventually(t, func() bool {
|
|
b, err := buffer.ReadBytes('\n')
|
|
if err != nil {
|
|
return false
|
|
}
|
|
assert.Equal(t, 4, len(b))
|
|
return true
|
|
}, 3*time.Second, 1*time.Second)
|
|
|
|
writer.Close()
|
|
assert.True(t, buffer.closed)
|
|
}
|