Documentation
¶
Index ¶
Constants ¶
This section is empty.
Variables ¶
View Source
var Composeyml = strings.TrimSpace(` version: '2' networks: airflow: driver: bridge volumes: postgres_data: driver: local airflow_logs: driver: local services: postgres: image: postgres:10.1-alpine restart: unless-stopped networks: - airflow labels: io.astronomer.docker: "true" io.astronomer.docker.cli: "true" ports: - {{ .PostgresPort }}:5432 volumes: - postgres_data:/var/lib/postgresql/data environment: POSTGRES_USER: {{ .PostgresUser }} POSTGRES_PASSWORD: {{ .PostgresPassword }} scheduler: image: {{ .AirflowImage }} command: ["airflow", "scheduler"] restart: unless-stopped networks: - airflow user: {{ .AirflowUser }} labels: io.astronomer.docker: "true" io.astronomer.docker.cli: "true" io.astronomer.docker.component: "airflow-scheduler" depends_on: - postgres environment: AIRFLOW__CORE__EXECUTOR: LocalExecutor AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql://{{ .PostgresUser }}:{{ .PostgresPassword }}@{{ .PostgresHost }}:5432 AIRFLOW__CORE__LOAD_EXAMPLES: "False" AIRFLOW__CORE__FERNET_KEY: "d6Vefz3G9U_ynXB3cr7y_Ak35tAHkEGAVxuz_B-jzWw=" volumes: - {{ .AirflowHome }}/dags:/usr/local/airflow/dags:ro - {{ .AirflowHome }}/plugins:/usr/local/airflow/plugins:ro - {{ .AirflowHome }}/include:/usr/local/airflow/include:ro - airflow_logs:/usr/local/airflow/logs {{ .AirflowEnvFile }} webserver: image: {{ .AirflowImage }} command: > bash -c "airflow create_user -r Admin -u admin -e [email protected] -f admin -l user -p admin && airflow webserver" restart: unless-stopped networks: - airflow user: {{ .AirflowUser }} labels: io.astronomer.docker: "true" io.astronomer.docker.cli: "true" io.astronomer.docker.component: "airflow-webserver" depends_on: - scheduler - postgres environment: AIRFLOW__CORE__EXECUTOR: LocalExecutor AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql://{{ .PostgresUser }}:{{ .PostgresPassword }}@{{ .PostgresHost }}:5432 AIRFLOW__CORE__LOAD_EXAMPLES: "False" AIRFLOW__CORE__FERNET_KEY: "d6Vefz3G9U_ynXB3cr7y_Ak35tAHkEGAVxuz_B-jzWw=" AIRFLOW__WEBSERVER__RBAC: "True" ports: - {{ .AirflowWebserverPort }}:8080 volumes: - {{ .AirflowHome }}/dags:/usr/local/airflow/dags:ro - {{ .AirflowHome }}/plugins:/usr/local/airflow/plugins:ro - {{ .AirflowHome }}/include:/usr/local/airflow/include:ro - airflow_logs:/usr/local/airflow/logs {{ .AirflowEnvFile }} `)
Composeyml is the docker-compose template
View Source
var Dockerfile = strings.TrimSpace(`
FROM astronomerinc/ap-airflow:%s
`)
Dockerfile is the Dockerfile template
View Source
var Dockerignore = strings.TrimSpace(`
.astro
.git
.env
airflow_setttings.yaml
`)
Dockerignore is the .dockerignore template
View Source
var ExamplePlugin = strings.TrimSpace(`
from airflow.plugins_manager import AirflowPlugin
from flask_admin.base import MenuLink
"""
Look for the Astronomer tab in the UI.
"""
airflow_plugins_ml = MenuLink(
category='Astronomer',
name='Airflow-Plugins',
url='https://github.com/airflow-plugins/')
astro_docs_ml = MenuLink(
category='Astronomer',
name='Astronomer Docs',
url='https://www.astronomer.io/docs/')
astro_guides_ml = MenuLink(
category='Astronomer',
name='Airflow Guides',
url='https://www.astronomer.io/guides/')
class AstroLinksPlugin(AirflowPlugin):
name = 'astronomer_menu_links'
operators = []
flask_blueprints = []
hooks = []
executors = []
macros = []
admin_views = []
menu_links = [airflow_plugins_ml, astro_docs_ml, astro_guides_ml]`)
ExamplePlugin created with astro airflow init
View Source
var Exampledag = strings.TrimSpace(`
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from datetime import datetime, timedelta
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': datetime(2018, 1, 1),
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
dag = DAG('example_dag',
max_active_runs=3,
schedule_interval=timedelta(minutes=5),
default_args=default_args)
t1 = BashOperator(
task_id='print_date1',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t2 = BashOperator(
task_id='print_date2',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t3 = BashOperator(
task_id='print_date3',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t4 = BashOperator(
task_id='print_date4',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t5 = BashOperator(
task_id='print_date5',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t6 = BashOperator(
task_id='print_date6',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t7 = BashOperator(
task_id='print_date7',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t8 = BashOperator(
task_id='print_date8',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t2.set_upstream(t1)
t3.set_upstream(t2)
t4.set_upstream(t3)
t5.set_upstream(t3)
t6.set_upstream(t3)
t7.set_upstream(t3)
t8.set_upstream(t3)
t9 = BashOperator(
task_id='print_date9',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t10 = BashOperator(
task_id='print_date10',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t11 = BashOperator(
task_id='print_date11',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t12 = BashOperator(
task_id='print_date12',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t9.set_upstream(t8)
t10.set_upstream(t8)
t11.set_upstream(t8)
t12.set_upstream(t8)
`)
Exampledag created with astro airflow init
View Source
var Gitignore = strings.TrimSpace(`
.astro
.git
.env
airflow_setttings.yaml
`)
Gitignore is the .gitignore template
View Source
var Settingsyml = strings.TrimSpace(`
# This feature is in Beta.
# Please report any bugs to https://github.com/astronomer/astro-cli/issues
# NOTE: If putting a dict in conn_extra, please wrap in single quotes.
# More details you can find https://github.com/astronomer/docs/blob/master/docs/cli-airflow-configuration.md
airflow:
connections:
- conn_id:
conn_type:
conn_host:
conn_login:
conn_password:
conn_port:
conn_extra:
pools:
- pool_name:
pool_slot:
pool_description:
variables:
- variable_name:
variable_value:`)
Settingsyml is the settings template
Functions ¶
This section is empty.
Types ¶
This section is empty.
Click to show internal directories.
Click to hide internal directories.