python -m venv .venv
source .venv/bin/activate
pip install -r requirements.txt
spark-submit spark/load.py local data_lake/1
# Practice 1
spark-submit spark/processing_daily_summary.py data_lake/1 data_lake/2_daily_summary
spark-submit spark/publish.py data_lake/2_daily_summary data_lake/3_daily_summary
# Practice 2
spark-submit spark/processing_yearly_summary.py data_lake/1 data_lake/2_yearly_summary
spark-submit spark/publish.py data_lake/2_yearly_summary data_lake/3_yearly_summary
# Practice 1
cp airflow/practice_1_local.py ~/airflow/dags/
# Practice 2
cp airflow/practice_2_local.py ~/airflow/dags/
Create cluster with bootstrap action: emr/bootstrap.sh
- Load -
$S3_BUCKET/script/load.py remote $S3_BUCKET/output/1
- Processing (Practice 1) -
spark-submit --deploy-mode cluster $S3_BUCKET/script/processing_daily_summary.py $S3_BUCKET/output/1 $S3_BUCKET/output/2_daily_summary
- Processing (Practice 2) -
spark-submit --deploy-mode cluster $S3_BUCKET/script/processing_yearly_summary.py $S3_BUCKET/output/1 $S3_BUCKET/output/2_yearly_summary
- Publish (Practice 1) -
spark-submit --deploy-mode cluster $S3_BUCKET/script/publish.py $S3_BUCKET/output/2_daily_summary $S3_BUCKET/output/3_daily_summary
- Publish (Practice 2) -
spark-submit --deploy-mode cluster $S3_BUCKET/script/publish.py $S3_BUCKET/output/2_yearly_summary $S3_BUCKET/output/3_yearly_summary