Documentation
¶
Index ¶
- Constants
- Variables
- func Banner()
- func CreateSeedJobs(langCode string, r io.Reader, maxDepth int, email bool, geoCoordinates string, ...) (jobs []scrapemate.IJob, err error)
- func LoadCustomWriter(pluginDir, pluginName string) (scrapemate.ResultWriter, error)
- func Telemetry() tlmt.Telemetry
- type Config
- type Runner
- type S3Uploader
Constants ¶
View Source
const ( RunModeFile = iota + 1 RunModeDatabase RunModeDatabaseProduce RunModeInstallPlaywright RunModeWeb RunModeAwsLambda RunModeAwsLambdaInvoker )
Variables ¶
View Source
var (
ErrInvalidRunMode = errors.New("invalid run mode")
)
Functions ¶
func CreateSeedJobs ¶
func LoadCustomWriter ¶
func LoadCustomWriter(pluginDir, pluginName string) (scrapemate.ResultWriter, error)
Types ¶
type Config ¶
type Config struct { Concurrency int CacheDir string MaxDepth int InputFile string ResultsFile string JSON bool LangCode string Debug bool Dsn string ProduceOnly bool ExitOnInactivityDuration time.Duration Email bool CustomWriter string GeoCoordinates string Zoom int RunMode int DisableTelemetry bool WebRunner bool AwsLamdbaRunner bool DataFolder string Proxies []string AwsAccessKey string AwsSecretKey string AwsRegion string S3Uploader S3Uploader S3Bucket string AwsLambdaInvoker bool FunctionName string AwsLambdaChunkSize int }
func ParseConfig ¶
func ParseConfig() *Config
Click to show internal directories.
Click to hide internal directories.