utils

func DecodeHTMLBody ¶

func DecodeHTMLBody(body io.Reader) (io.Reader, error)

DecodeHTMLBody returns an decoding reader of the html Body for the specified `charset` If `charset` is empty, DecodeHTMLBody tries to guess the encoding from the content

func GetHostByURL ¶

func GetHostByURL(url string) string

func GetLogger ¶

func GetLogger() *logger.Logger

func GetPage ¶

func GetPage(url, ua string) (io.Reader, error)

func IsExist ¶

func IsExist(fileName string) bool

func Nodetext ¶

func Nodetext(node *html.Node) string

func ParseRule ¶

func ParseRule(rule string) (string, int, int)

return selector,length of rules, index of selector

func ParseRules ¶

func ParseRules(doc interface{}, rule string) (*goquery.Selection, string)

func PostPage ¶

func PostPage(url, key, ua string) (io.Reader, error)

func RemoveNodes ¶

func RemoveNodes(srcNodes, removeNodes []*html.Node) []*html.Node

func URLFix ¶

func URLFix(uri, host string) string