diff --git a/Makefile b/Makefile index 6b0575c..e9c3566 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ sh: docker exec -ti local-spark bash meta: - PGPASSWORD=sdepassword pgcli -h localhost -p 5432 -U sdeuser -d metadatadb + PGPASSWORD=airflow pgcli -h localhost -p 5432 -U airflow -d airflow ddl: docker exec -ti local-spark bash -c '$$SPARK_HOME/bin/spark-submit --master local[*] --packages io.delta:$${DELTA_PACKAGE_VERSION},org.apache.hadoop:hadoop-aws:3.3.2 --conf spark.databricks.delta.retentionDurationCheck.enabled=false --conf spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension --conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog --conf spark.hadoop.fs.s3a.access.key=minio --conf spark.hadoop.fs.s3a.secret.key=minio123 --conf spark.hadoop.fs.s3a.endpoint=http://minio:9000 --conf spark.hadoop.fs.s3a.region=us-east-1 --conf spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem --conf spark.hadoop.fs.s3a.path.style.access=true ./adventureworks/ddl/create_tables.py' diff --git a/docker-compose.yml b/docker-compose.yml index 137d9a4..7c0509c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3' - x-airflow-common: &airflow-common build: