Documentation
¶
Index ¶
- Constants
- func Count(fd FileObjectModel) (int64, error)
- func Merge(src []FileObjectModel, dest FileObjectModel) error
- func RegisterChecksumer(key string, ch Checksumer)
- func RegisterFileType(ftype string, reader FileItemReader, writer FileItemWriter, ...)
- func Split(srcFd FileObjectModel, destFds []FileObjectModel, strategy FileSplitStrategy) error
- type ChecksumFlusher
- type ChecksumVerifier
- type Checksumer
- type FTPFileInStream
- type FTPFileOutStream
- type FTPFileSystem
- type FileItemReader
- type FileItemWriter
- type FileMerger
- type FileMove
- type FileObjectModel
- type FileSplitStrategy
- type FileSplitter
- type FileStorage
- type LocalFileSystem
- type MD5Checksumer
- type MergeSplitter
- type OKFlagChecksumer
- type SHA1Checksumer
- type SHA256Checksumer
- type SHA512Checksumer
Constants ¶
const ( //LocalFileStorage name of local file storage LocalFileStorage = "File" //FTPFileStorage name of ftp file storage FTPFileStorage = "FTP" )
const ( //TSV a file type that have tab-seperated values as its content TSV = "tsv" //CSV a file type that have comma-seperated values as its content CSV = "csv" //JSON a file type that have json string as its content JSON = "json" )
const ( OKFlag = "OK" MD5 = "MD5" SHA1 = "SHA1" SHA256 = "SHA256" SHA512 = "SHA512" )
Variables ¶
This section is empty.
Functions ¶
func Count ¶
func Count(fd FileObjectModel) (int64, error)
Count the default implementation to count number of items contained by the file
func Merge ¶
func Merge(src []FileObjectModel, dest FileObjectModel) error
Merge the default implementation to merge multiple files into a simple one
func RegisterChecksumer ¶
func RegisterChecksumer(key string, ch Checksumer)
func RegisterFileType ¶
func RegisterFileType(ftype string, reader FileItemReader, writer FileItemWriter, mergeSplitter MergeSplitter)
RegisterFileType register file type
func Split ¶
func Split(srcFd FileObjectModel, destFds []FileObjectModel, strategy FileSplitStrategy) error
Split the default implementation to split a file into multiple destinations
Types ¶
type ChecksumFlusher ¶
type ChecksumFlusher interface { //Checksum generate checksum and write a check file for the corresponding data file Checksum(fd FileObjectModel) error }
ChecksumFlusher write a check file for the corresponding data file
type ChecksumVerifier ¶
type ChecksumVerifier interface { //Verify verify a data file according to the corresponding check file Verify(fd FileObjectModel) (bool, error) }
ChecksumVerifier verify a data file according to the corresponding check file
type Checksumer ¶
type Checksumer interface { ChecksumVerifier ChecksumFlusher }
Checksumer the combination of ChecksumVerifier and ChecksumFlusher
type FTPFileInStream ¶
type FTPFileInStream struct {
// contains filtered or unexported fields
}
FTPFileInStream represents a ftp file input stream
func (*FTPFileInStream) Close ¶
func (fis *FTPFileInStream) Close() (err error)
type FTPFileOutStream ¶
type FTPFileOutStream struct {
// contains filtered or unexported fields
}
FTPFileOutStream represents a ftp file output stream
func (*FTPFileOutStream) Close ¶
func (fis *FTPFileOutStream) Close() (err error)
type FTPFileSystem ¶
type FTPFileSystem struct { Hort string Port int User string Password string ConnTimeout time.Duration }
FTPFileSystem a FileStorage implementation backed by ftp
func (*FTPFileSystem) Create ¶
func (fs *FTPFileSystem) Create(fileName, encoding string) (writer io.WriteCloser, err error)
func (*FTPFileSystem) Name ¶
func (fs *FTPFileSystem) Name() string
func (*FTPFileSystem) Open ¶
func (fs *FTPFileSystem) Open(fileName, encoding string) (io.ReadCloser, error)
type FileItemReader ¶
type FileItemReader interface { //Open open file of the FileObjectModel, returns a handle for later read Open(fd FileObjectModel) (handle interface{}, err error) //Close open the file handle Close(handle interface{}) error //ReadItem read an item from the file handle ReadItem(handle interface{}) (interface{}, error) //SkipTo skip to the pos'th record in the file SkipTo(handle interface{}, pos int64) error //Count return number of items contained by the file Count(fd FileObjectModel) (int64, error) }
FileItemReader read object from file according a FileObjectModel
func GetFileItemReader ¶
func GetFileItemReader(ftype string) FileItemReader
GetFileItemReader get FileItemReader by type
type FileItemWriter ¶
type FileItemWriter interface { //Open open file of the FileObjectModel, returns a handle for later write Open(fd FileObjectModel) (handle interface{}, err error) //Close open the file handle Close(handle interface{}) error //WriteItem write an item to the file WriteItem(handle interface{}, data interface{}) error }
FileItemWriter write object to file according to a FileObjectModel
func GetFileItemWriter ¶
func GetFileItemWriter(ftype string) FileItemWriter
GetFileItemWriter get FileItemWriter by type
type FileMerger ¶
type FileMerger interface { //Merge merge multiple files into a simple one Merge(src []FileObjectModel, dest FileObjectModel) (err error) }
FileMerger merge multiple files into a simple one
type FileMove ¶
type FileMove struct { FromFileName string FromFileStore FileStorage ToFileName string ToFileStore FileStorage }
FileMove represents source and destination of a file movement
type FileObjectModel ¶
type FileObjectModel struct { FileStore FileStorage FileName string Type string Encoding string Header bool FieldSeparator string Checksum string ItemPrototype interface{} }
FileObjectModel represents mapping a file to/from a struct
func (FileObjectModel) ItemType ¶
func (fd FileObjectModel) ItemType() (reflect.Type, error)
ItemType returns the reflect.Type of the item within a FileObjectModel
func (FileObjectModel) String ¶
func (fd FileObjectModel) String() string
type FileSplitStrategy ¶
type FileSplitStrategy interface { //DecideDestIndex decide the destination of every record, return the index of destination file in the dest array DecideDestIndex(line string, dest []FileObjectModel) int }
FileSplitStrategy decide the destination of every record
type FileSplitter ¶
type FileSplitter interface { //Split split a src file into multiple destinations, decide the destination of every record according to a FileSplitStrategy Split(src FileObjectModel, dest []FileObjectModel, strategy FileSplitStrategy) (err error) }
FileSplitter split a src file into multiple dest ones
type FileStorage ¶
type FileStorage interface { //Name return name of the file store Name() string //Exists check if a file with the specified fileName exists Exists(fileName string) (ok bool, err error) //Open return a reader of the specified fileName, transform encoding if necessary Open(fileName, encoding string) (reader io.ReadCloser, err error) //Create return a writer of the specified fileName, transform encoding if necessary Create(fileName, encoding string) (writer io.WriteCloser, err error) }
FileStorage represents a file store media, from which we can read or write files
type LocalFileSystem ¶
type LocalFileSystem struct { }
LocalFileSystem a FileStorage implementation backed by local file system
func (*LocalFileSystem) Create ¶
func (fs *LocalFileSystem) Create(fileName, encoding string) (io.WriteCloser, error)
func (*LocalFileSystem) Name ¶
func (fs *LocalFileSystem) Name() string
func (*LocalFileSystem) Open ¶
func (fs *LocalFileSystem) Open(fileName, encoding string) (io.ReadCloser, error)
type MD5Checksumer ¶
type MD5Checksumer struct { }
MD5Checksumer generate and verify a check file containing the md5 digest of the data file
func (*MD5Checksumer) Checksum ¶
func (ch *MD5Checksumer) Checksum(fd FileObjectModel) error
func (*MD5Checksumer) Verify ¶
func (ch *MD5Checksumer) Verify(fd FileObjectModel) (bool, error)
type MergeSplitter ¶
type MergeSplitter interface { FileMerger FileSplitter }
MergeSplitter the combination of FileMerger and FileSplitter
func GetFileMergeSplitter ¶
func GetFileMergeSplitter(ftype string) MergeSplitter
GetFileMergeSplitter get MergeSplitter by type
type OKFlagChecksumer ¶
type OKFlagChecksumer struct { }
OKFlagChecksumer generate and verify an empty file with '.ok' suffix indicating the data file completed
func (*OKFlagChecksumer) Checksum ¶
func (ch *OKFlagChecksumer) Checksum(fd FileObjectModel) error
func (*OKFlagChecksumer) Verify ¶
func (ch *OKFlagChecksumer) Verify(fd FileObjectModel) (bool, error)
type SHA1Checksumer ¶
type SHA1Checksumer struct { }
SHA1Checksumer generate and verify a check file containing the sha-1 digest of the data file
func (*SHA1Checksumer) Checksum ¶
func (ch *SHA1Checksumer) Checksum(fd FileObjectModel) error
func (*SHA1Checksumer) Verify ¶
func (ch *SHA1Checksumer) Verify(fd FileObjectModel) (bool, error)
type SHA256Checksumer ¶
type SHA256Checksumer struct { }
SHA256Checksumer generate and verify a check file containing the sha-256 digest of the data file
func (*SHA256Checksumer) Checksum ¶
func (ch *SHA256Checksumer) Checksum(fd FileObjectModel) error
func (*SHA256Checksumer) Verify ¶
func (ch *SHA256Checksumer) Verify(fd FileObjectModel) (bool, error)
type SHA512Checksumer ¶
type SHA512Checksumer struct { }
SHA512Checksumer generate and verify a check file containing the sha-512 digest of the data file
func (*SHA512Checksumer) Checksum ¶
func (ch *SHA512Checksumer) Checksum(fd FileObjectModel) error
func (*SHA512Checksumer) Verify ¶
func (ch *SHA512Checksumer) Verify(fd FileObjectModel) (bool, error)