From 9ef13a66ecd5810ee3a47641d3de96998afaf270 Mon Sep 17 00:00:00 2001 From: ia-tests Date: Thu, 23 Jul 2020 13:23:58 -0700 Subject: [PATCH 1/2] patch sql path rendering --- composer/dags/wordcount_dag.py | 5 ++--- helpers/run_tests.sh | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/composer/dags/wordcount_dag.py b/composer/dags/wordcount_dag.py index 9423570..a6cd870 100644 --- a/composer/dags/wordcount_dag.py +++ b/composer/dags/wordcount_dag.py @@ -16,7 +16,7 @@ import datetime import os -from airflow import models +from airflow import models, settings from airflow.contrib.operators.bigquery_operator import BigQueryOperator from airflow.contrib.operators.dataflow_operator import DataFlowJavaOperator from airflow.contrib.operators.gcs_download_operator import \ @@ -40,8 +40,7 @@ # Dynamic prefix gives us flexibility for running airflow in a ci container or # on composer. -SQL_PREFIX = os.path.join(os.environ.get('AIRFLOW_HOME', '/home/airflow'), - 'gcs', 'data', 'sql') +SQL_PREFIX = os.path.join(settings.DAGS_FOLDER, 'sql') SHAKESPEARE_SQL = os.path.join(SQL_PREFIX, 'shakespeare_top_25.sql') diff --git a/helpers/run_tests.sh b/helpers/run_tests.sh index b2f0798..3250f76 100755 --- a/helpers/run_tests.sh +++ b/helpers/run_tests.sh @@ -25,3 +25,6 @@ fi echo "running dataflow java tests..." find ./dataflow/java/ -name pom.xml -execdir mvn test \; + +echo "running airflow tests" +(cd composer && ./cloudbuild/bin/run_tests.sh ../bigquery/sql ./config/AirflowVariables.json ./plugins) From f40a4b120cff7790ec90514f416030f858ae60d9 Mon Sep 17 00:00:00 2001 From: ia-tests Date: Thu, 23 Jul 2020 13:34:29 -0700 Subject: [PATCH 2/2] add dags --- composer/config/running_dags.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/composer/config/running_dags.txt b/composer/config/running_dags.txt index e69de29..a938855 100644 --- a/composer/config/running_dags.txt +++ b/composer/config/running_dags.txt @@ -0,0 +1,2 @@ +ephemeral_dataproc_spark_dag +wordcount_dag