We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 59461c9 commit 328740fCopy full SHA for 328740f
cob_datapipeline/scripts/ingest_gencon.sh
@@ -2,19 +2,12 @@
2
3
set -eo pipefail
4
5
-# export / set all environment variables passed here by task for pick-up by subprocess
6
-export AIRFLOW_USER_HOME="/home/airflow"
7
-export HOME=$AIRFLOW_USER_HOME
8
-
9
source $HOME/.bashrc
10
11
-export PATH="$AIRFLOW_USER_HOME/.local/bin:$PATH"
12
-export PATH="$AIRFLOW_USER_HOME/.rbenv/shims:$AIRFLOW_USER_HOME/.rbenv/bin:$PATH"
13
14
export SOLR_URL="${SOLR_WEB_URL//\/\////$SOLR_AUTH_USER:$SOLR_AUTH_PASSWORD@}"
15
-export GENCON_INDEX_PATH="$PWD/gencon_index"
16
17
# Get the raw CSV files from S3
+mkdir -p $GENCON_TEMP_PATH
18
aws s3 sync $GENCON_CSV_S3 $GENCON_TEMP_PATH --include "*.csv"
19
20
if [[ ! -d "$GENCON_INDEX_PATH" ]]; then
0 commit comments