代码拉取完成,页面将自动刷新
同步操作将从 chararch/gobatch 强制同步,此操作会覆盖自 Fork 仓库以来所做的任何修改,且无法恢复!!!
确定后同步将在后台操作,完成时将刷新页面,请耐心等待。
package gobatch
import (
"context"
"fmt"
"github.com/chararch/gobatch/file"
)
const (
fileItemReaderHandleKey = "gobatch.FileItemReader.handle"
fileItemReaderFileNameKey = "gobatch.FileItemWriter.fileName"
fileItemReaderCurrentIndex = "gobatch.FileItemReader.current.index"
fileItemReaderStart = "gobatch.FileItemReader.start"
fileItemReaderEnd = "gobatch.FileItemReader.end"
)
type fileReader struct {
fd file.FileObjectModel
reader file.FileItemReader
verifier file.ChecksumVerifier
}
func (r *fileReader) Open(execution *StepExecution) BatchError {
//get actual file name
fd := r.fd //copy fd
fp := &FilePath{fd.FileName}
fileName, err := fp.Format(execution)
if err != nil {
return NewBatchError(ErrCodeGeneral, "get real file path:%v err", fd.FileName, err)
}
fd.FileName = fileName
//verify checksum
if fd.Checksum != "" {
checksumer := file.GetChecksumer(fd.Checksum)
if checksumer != nil {
ok, err := checksumer.Verify(fd)
if err != nil || !ok {
return NewBatchError(ErrCodeGeneral, "verify file checksum:%v, ok:%v err", fd, ok, err)
}
}
}
//read file
handle, e := r.reader.Open(fd)
if e != nil {
return NewBatchError(ErrCodeGeneral, "open file reader:%v err", fd, e)
}
execution.StepExecutionContext.Put(fileItemReaderHandleKey, handle)
execution.StepExecutionContext.Put(fileItemReaderFileNameKey, fd.FileName)
executionCtx := execution.StepExecutionContext
currentIndex, _ := executionCtx.GetInt64(fileItemReaderCurrentIndex)
err = r.reader.SkipTo(handle, currentIndex)
if err != nil {
return NewBatchError(ErrCodeGeneral, "skip to file item:%v pos:%v err", fd, currentIndex, err)
}
return nil
}
func (r *fileReader) Read(chunkCtx *ChunkContext) (interface{}, BatchError) {
stepCtx := chunkCtx.StepExecution.StepContext
executionCtx := chunkCtx.StepExecution.StepExecutionContext
endPos, _ := stepCtx.GetInt64(fileItemReaderEnd)
currentIndex, _ := executionCtx.GetInt64(fileItemReaderCurrentIndex)
handle := executionCtx.Get(fileItemReaderHandleKey)
fileName := executionCtx.Get(fileItemReaderFileNameKey)
if currentIndex < endPos {
item, e := r.reader.ReadItem(handle)
if e != nil {
return nil, NewBatchError(ErrCodeGeneral, "read item from file:%v err", fileName, e)
}
executionCtx.Put(fileItemReaderCurrentIndex, currentIndex+1)
return item, nil
}
return nil, nil
}
func (r *fileReader) Close(execution *StepExecution) BatchError {
executionCtx := execution.StepExecutionContext
handle := executionCtx.Get(fileItemReaderHandleKey)
fileName := executionCtx.Get(fileItemReaderFileNameKey)
executionCtx.Remove(fileItemReaderHandleKey)
e := r.reader.Close(handle)
if e != nil {
return NewBatchError(ErrCodeGeneral, "close file reader:%v err", fileName, e)
}
return nil
}
func (r *fileReader) GetPartitioner(minPartitionSize, maxPartitionSize uint) Partitioner {
return &filePartitioner{
fd: r.fd,
reader: r.reader,
minPartitionSize: minPartitionSize,
maxPartitionSize: maxPartitionSize,
}
}
type filePartitioner struct {
fd file.FileObjectModel
reader file.FileItemReader
minPartitionSize uint
maxPartitionSize uint
}
func (p *filePartitioner) Partition(execution *StepExecution, partitions uint) (subExecutions []*StepExecution, e BatchError) {
defer func() {
if err := recover(); err != nil {
e = NewBatchError(ErrCodeGeneral, "panic on Partition in filePartitioner, err", err)
}
}()
// get actual file name
fd := p.fd //copy fd
fp := &FilePath{fd.FileName}
fileName, err := fp.Format(execution)
if err != nil {
return nil, NewBatchError(ErrCodeGeneral, "get real file path:%v err", fd.FileName, err)
}
fd.FileName = fileName
//verify checksum
if fd.Checksum != "" {
checksumer := file.GetChecksumer(fd.Checksum)
if checksumer != nil {
ok, err := checksumer.Verify(fd)
if err != nil || !ok {
return nil, NewBatchError(ErrCodeGeneral, "verify file checksum:%v, ok:%v err", fd, ok, err)
}
}
}
//read file
count, err := p.reader.Count(fd)
if err != nil {
return nil, NewBatchError(ErrCodeGeneral, "Count() err", err)
}
subExecutions = make([]*StepExecution, 0)
if count == 0 {
return subExecutions, nil
}
partitionSize := uint(count) / partitions
if partitionSize > p.maxPartitionSize {
partitionSize = p.maxPartitionSize
}
if partitionSize < p.minPartitionSize {
partitionSize = p.minPartitionSize
}
i := uint(0)
for start, end := int64(0), int64(partitionSize); start < count; start, end = end, end+int64(partitionSize) {
if end > count {
end = count
}
partitionName := genPartitionStepName(execution, i)
subExecution := execution.deepCopy()
subExecution.StepName = partitionName
subExecution.StepContextId = 0
subExecution.StepContext.Put(fileItemReaderStart, start)
subExecution.StepContext.Put(fileItemReaderEnd, end)
subExecution.StepExecutionContext.Put(fileItemReaderCurrentIndex, start)
subExecutions = append(subExecutions, subExecution)
i++
}
logger.Info(context.Background(), "partition step:%v, total count:%v, partitions:%v, partitionSize:%v, subExecutions:%v", execution.StepName, count, partitions, partitionSize, len(subExecutions))
return subExecutions, nil
}
func genPartitionStepName(execution *StepExecution, i uint) string {
partitionName := fmt.Sprintf("%s:%04d", execution.StepName, i)
return partitionName
}
func (p *filePartitioner) GetPartitionNames(execution *StepExecution, partitions uint) []string {
names := make([]string, 0)
for i := uint(0); i < partitions; i++ {
partitionName := genPartitionStepName(execution, i)
names = append(names, partitionName)
}
return names
}
此处可能存在不合适展示的内容,页面不予展示。您可通过相关编辑功能自查并修改。
如您确认内容无涉及 不当用语 / 纯广告导流 / 暴力 / 低俗色情 / 侵权 / 盗版 / 虚假 / 无价值内容或违法国家有关法律法规的内容,可点击提交进行申诉,我们将尽快为您处理。