runner

package
v1.7.10 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 22, 2025 License: MIT Imports: 24 Imported by: 0

Documentation

Index

Constants

View Source
const (
	RunModeFile = iota + 1
	RunModeDatabase
	RunModeDatabaseProduce
	RunModeInstallPlaywright
	RunModeWeb
	RunModeAwsLambda
	RunModeAwsLambdaInvoker
)

Variables

View Source
var (
	ErrInvalidRunMode = errors.New("invalid run mode")
)

Functions

func Banner()

func CreateSeedJobs

func CreateSeedJobs(
	fastmode bool,
	langCode string,
	r io.Reader,
	maxDepth int,
	email bool,
	geoCoordinates string,
	zoom int,
	radius float64,
	dedup deduper.Deduper,
	exitMonitor exiter.Exiter,
) (jobs []scrapemate.IJob, err error)

func LoadCustomWriter

func LoadCustomWriter(pluginDir, pluginName string) (scrapemate.ResultWriter, error)

func Telemetry added in v1.3.4

func Telemetry() tlmt.Telemetry

Types

type Config

type Config struct {
	Concurrency              int
	CacheDir                 string
	MaxDepth                 int
	InputFile                string
	ResultsFile              string
	JSON                     bool
	LangCode                 string
	Debug                    bool
	Dsn                      string
	ProduceOnly              bool
	ExitOnInactivityDuration time.Duration
	Email                    bool
	CustomWriter             string
	GeoCoordinates           string
	Zoom                     int
	RunMode                  int
	DisableTelemetry         bool
	WebRunner                bool
	AwsLamdbaRunner          bool
	DataFolder               string
	Proxies                  []string
	AwsAccessKey             string
	AwsSecretKey             string
	AwsRegion                string
	S3Uploader               S3Uploader
	S3Bucket                 string
	AwsLambdaInvoker         bool
	FunctionName             string
	AwsLambdaChunkSize       int
	FastMode                 bool
	Radius                   float64
	Addr                     string
	DisablePageReuse         bool
}

func ParseConfig

func ParseConfig() *Config

type Runner

type Runner interface {
	Run(context.Context) error
	Close(context.Context) error
}

type S3Uploader added in v1.6.0

type S3Uploader interface {
	Upload(ctx context.Context, bucketName, key string, body io.Reader) error
}

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
JackTT - Gopher 🇻🇳