crawler

package

v0.1.1-0...-c572a67 Latest Latest Go to latest Published: May 22, 2018 License: Apache-2.0 Imports: 12 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/crawlerclub/x

Links

Open Source Insights

Documentation ¶

Constants ¶

This section is empty.

Variables ¶

View Source

var (
	ErrEmptyCrawlerConf = errors.New("crawler/crawler.go empty CrawlerConf")
	ErrNilTask          = errors.New("crawler/crawler.go nil Task")
	ErrNilItem          = errors.New("crawler/crawler.go nil Item")
	ErrNilTaskQueue     = errors.New("crawler/crawler.go nil TaskQueue")
)

Functions ¶

This section is empty.

Types ¶

type Crawler ¶

type Crawler struct {
	Conf      *types.CrawlerConf
	TaskQueue *ds.Queue
	// contains filtered or unexported fields
}

func (*Crawler) Close ¶

func (self *Crawler) Close()

func (*Crawler) InitEs ¶

func (self *Crawler) InitEs() error

func (*Crawler) InitTaskQueue ¶

func (self *Crawler) InitTaskQueue(dir string) error

func (*Crawler) LoadConfFromBytes ¶

func (self *Crawler) LoadConfFromBytes(str []byte) error

func (*Crawler) LoadConfFromFile ¶

func (self *Crawler) LoadConfFromFile(file string) error

func (*Crawler) Process ¶

func (self *Crawler) Process(
	task *types.Task) ([]types.Task, []map[string]interface{}, error)

func (*Crawler) Save ¶

func (self *Crawler) Save(item map[string]interface{}) error

func (*Crawler) Test ¶

func (self *Crawler) Test() (map[string]interface{}, error)

Source Files ¶

View all Source files

crawler.go

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL