From 3d8f84935fc46343c5c7e95b88358bf71676d734 Mon Sep 17 00:00:00 2001 From: alrocar Date: Tue, 30 Jul 2024 15:06:03 +0200 Subject: [PATCH 1/9] udpate ci --- .github/workflows/iterating_api_endpoint_ci.yml | 3 ++- iterating_api_endpoint/includes/top_pages_test.incl | 3 +++ iterating_api_endpoint/pipes/top_pages.pipe | 7 +++++-- 3 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 iterating_api_endpoint/includes/top_pages_test.incl diff --git a/.github/workflows/iterating_api_endpoint_ci.yml b/.github/workflows/iterating_api_endpoint_ci.yml index d78480be..e8e23733 100644 --- a/.github/workflows/iterating_api_endpoint_ci.yml +++ b/.github/workflows/iterating_api_endpoint_ci.yml @@ -20,9 +20,10 @@ jobs: ci: # ci using environments from workspace 'iterating_api_endpoint' - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 + uses: tinybirdco/ci/.github/workflows/ci.yml@support-env with: data_project_dir: iterating_api_endpoint + tb_env: test secrets: tb_admin_token: ${{ secrets.ITERATING_API_ENDPOINT_TOKEN }} # set the Workspace admin token in GitHub secrets tb_host: https://api.tinybird.co diff --git a/iterating_api_endpoint/includes/top_pages_test.incl b/iterating_api_endpoint/includes/top_pages_test.incl new file mode 100644 index 00000000..ed5ceef5 --- /dev/null +++ b/iterating_api_endpoint/includes/top_pages_test.incl @@ -0,0 +1,3 @@ +NODE include_top_pages +SQL > + SELECT 1 diff --git a/iterating_api_endpoint/pipes/top_pages.pipe b/iterating_api_endpoint/pipes/top_pages.pipe index 394f362d..88650493 100644 --- a/iterating_api_endpoint/pipes/top_pages.pipe +++ b/iterating_api_endpoint/pipes/top_pages.pipe @@ -6,6 +6,8 @@ DESCRIPTION > TOKEN "dashboard" READ +INCLUDE "../includes/top_pages_$TB_ENV.incl" + NODE endpoint DESCRIPTION > Group by pagepath and calcualte hits and visits @@ -16,7 +18,8 @@ SQL > select pathname, uniqMerge(visits) as visits, - countMerge(hits) as hits + countMerge(hits) as hits, + 1 as $TB_ENV from analytics_pages_mv where @@ -31,7 +34,7 @@ SQL > and date <= today() {% end %} group by - pathname + pathname, $TB_ENV order by visits desc limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} From 8eff8c93b7f702cb79701865f1be8088f7d38e0d Mon Sep 17 00:00:00 2001 From: alrocar Date: Tue, 30 Jul 2024 15:28:46 +0200 Subject: [PATCH 2/9] use main --- .github/workflows/iterating_api_endpoint_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/iterating_api_endpoint_ci.yml b/.github/workflows/iterating_api_endpoint_ci.yml index e8e23733..0fe9a882 100644 --- a/.github/workflows/iterating_api_endpoint_ci.yml +++ b/.github/workflows/iterating_api_endpoint_ci.yml @@ -20,7 +20,7 @@ jobs: ci: # ci using environments from workspace 'iterating_api_endpoint' - uses: tinybirdco/ci/.github/workflows/ci.yml@support-env + uses: tinybirdco/ci/.github/workflows/ci.yml@main with: data_project_dir: iterating_api_endpoint tb_env: test From e6eae3977b6d15a2ec9c0c42bd99c591cfa726a5 Mon Sep 17 00:00:00 2001 From: alrocar Date: Tue, 30 Jul 2024 15:31:12 +0200 Subject: [PATCH 3/9] use 4.1.0 --- .github/workflows/iterating_api_endpoint_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/iterating_api_endpoint_ci.yml b/.github/workflows/iterating_api_endpoint_ci.yml index 0fe9a882..31111561 100644 --- a/.github/workflows/iterating_api_endpoint_ci.yml +++ b/.github/workflows/iterating_api_endpoint_ci.yml @@ -20,7 +20,7 @@ jobs: ci: # ci using environments from workspace 'iterating_api_endpoint' - uses: tinybirdco/ci/.github/workflows/ci.yml@main + uses: tinybirdco/ci/.github/workflows/ci.yml@v4.1.0 with: data_project_dir: iterating_api_endpoint tb_env: test From bb8456d01a3a8b0aab281b4ee46d9887b3aeab78 Mon Sep 17 00:00:00 2001 From: alrocar Date: Wed, 31 Jul 2024 10:33:04 +0200 Subject: [PATCH 4/9] configure kafka with env --- .../workflows/iterating_api_endpoint_cd.yml | 3 +- .../iterating_api_endpoint_cd_prod.yml | 25 ++++++++++++++++ .../workflows/iterating_api_endpoint_ci.yml | 2 +- .../iterating_api_endpoint_ci_prod.yml | 29 +++++++++++++++++++ .../datasources/kafka.datasource | 22 ++++++++++++++ .../includes/kafka_prod.incl | 8 +++++ .../{top_pages_test.incl => kafka_stg.incl} | 2 +- 7 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/iterating_api_endpoint_cd_prod.yml create mode 100644 .github/workflows/iterating_api_endpoint_ci_prod.yml create mode 100644 iterating_api_endpoint/datasources/kafka.datasource create mode 100644 iterating_api_endpoint/includes/kafka_prod.incl rename iterating_api_endpoint/includes/{top_pages_test.incl => kafka_stg.incl} (59%) diff --git a/.github/workflows/iterating_api_endpoint_cd.yml b/.github/workflows/iterating_api_endpoint_cd.yml index bc60e286..20a72ce6 100644 --- a/.github/workflows/iterating_api_endpoint_cd.yml +++ b/.github/workflows/iterating_api_endpoint_cd.yml @@ -16,9 +16,10 @@ - 'iterating_api_endpoint/**' jobs: cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 + uses: tinybirdco/ci/.github/workflows/cd.yml@v4.1.0 with: data_project_dir: iterating_api_endpoint + tb_env: stg secrets: tb_admin_token: ${{ secrets.ITERATING_API_ENDPOINT_TOKEN }} # set the Workspace admin token in GitHub secrets tb_host: https://api.tinybird.co diff --git a/.github/workflows/iterating_api_endpoint_cd_prod.yml b/.github/workflows/iterating_api_endpoint_cd_prod.yml new file mode 100644 index 00000000..1c2f701e --- /dev/null +++ b/.github/workflows/iterating_api_endpoint_cd_prod.yml @@ -0,0 +1,25 @@ + + ################################################## + ### Visit https://github.com/tinybirdco/ci ### + ### for more details or custom CI/CD ### + ################################################## + + name: Tinybird - CD Workflow + + on: + workflow_dispatch: + push: + branches: + - main + - master + paths: + - 'iterating_api_endpoint/**' + jobs: + cd: + uses: tinybirdco/ci/.github/workflows/cd.yml@v4.1.0 + with: + data_project_dir: iterating_api_endpoint + tb_env: prod + secrets: + tb_admin_token: ${{ secrets.ITERATING_API_ENDPOINT_TOKEN_PROD }} # set the Workspace admin token in GitHub secrets + tb_host: https://api.tinybird.co diff --git a/.github/workflows/iterating_api_endpoint_ci.yml b/.github/workflows/iterating_api_endpoint_ci.yml index 31111561..9fc27f4e 100644 --- a/.github/workflows/iterating_api_endpoint_ci.yml +++ b/.github/workflows/iterating_api_endpoint_ci.yml @@ -23,7 +23,7 @@ uses: tinybirdco/ci/.github/workflows/ci.yml@v4.1.0 with: data_project_dir: iterating_api_endpoint - tb_env: test + tb_env: stg secrets: tb_admin_token: ${{ secrets.ITERATING_API_ENDPOINT_TOKEN }} # set the Workspace admin token in GitHub secrets tb_host: https://api.tinybird.co diff --git a/.github/workflows/iterating_api_endpoint_ci_prod.yml b/.github/workflows/iterating_api_endpoint_ci_prod.yml new file mode 100644 index 00000000..73ccaa9e --- /dev/null +++ b/.github/workflows/iterating_api_endpoint_ci_prod.yml @@ -0,0 +1,29 @@ + + ################################################## + ### Visit https://github.com/tinybirdco/ci ### + ### for more details or custom CI/CD ### + ################################################## + + name: Tinybird - CI Workflow + + on: + workflow_dispatch: + pull_request: + branches: + - main + - master + types: [opened, reopened, labeled, unlabeled, synchronize, closed] + paths: + - 'iterating_api_endpoint/**' + + concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} + + jobs: + ci: # ci using environments from workspace 'iterating_api_endpoint' + uses: tinybirdco/ci/.github/workflows/ci.yml@v4.1.0 + with: + data_project_dir: iterating_api_endpoint + tb_env: prod + secrets: + tb_admin_token: ${{ secrets.ITERATING_API_ENDPOINT_TOKEN_PROD }} # set the Workspace admin token in GitHub secrets + tb_host: https://api.tinybird.co diff --git a/iterating_api_endpoint/datasources/kafka.datasource b/iterating_api_endpoint/datasources/kafka.datasource new file mode 100644 index 00000000..371507c3 --- /dev/null +++ b/iterating_api_endpoint/datasources/kafka.datasource @@ -0,0 +1,22 @@ +SCHEMA > + `datetime` DateTime `json:$.datetime`, + `device` String `json:$.device`, + `event_id` String `json:$.event_id`, + `event_name` String `json:$.event_name`, + `fingerprint` String `json:$.fingerprint`, + `price` Int16 `json:$.price`, + `product_id` String `json:$.product_id`, + `product_type` String `json:$.product_type`, + `url` String `json:$.url`, + `user_agent` String `json:$.user_agent`, + `meta_color` Nullable(String) `json:$.meta.color`, + `meta_size` Nullable(String) `json:$.meta.size`, + `meta_image` Nullable(String) `json:$.meta.image`, + `meta_image_v2` Nullable(String) `json:$.meta.image_v2`, + `meta_image_v3` Nullable(String) `json:$.meta.image_v3` + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(__timestamp)" +ENGINE_SORTING_KEY "__timestamp" + +INCLUDE "../includes/kafka_$TB_ENV.incl" \ No newline at end of file diff --git a/iterating_api_endpoint/includes/kafka_prod.incl b/iterating_api_endpoint/includes/kafka_prod.incl new file mode 100644 index 00000000..954dc70d --- /dev/null +++ b/iterating_api_endpoint/includes/kafka_prod.incl @@ -0,0 +1,8 @@ +KAFKA_CONNECTION_NAME 'kafka_connection_prod' +KAFKA_TOPIC 'add_column_kafka_ds_prod' +KAFKA_GROUP_ID 'add_column_kafka_ds_ex101_prod' +KAFKA_AUTO_OFFSET_RESET 'earliest' +KAFKA_STORE_RAW_VALUE 'False' +KAFKA_STORE_HEADERS 'False' +KAFKA_TARGET_PARTITIONS 'auto' +KAFKA_KEY_AVRO_DESERIALIZATION '' \ No newline at end of file diff --git a/iterating_api_endpoint/includes/top_pages_test.incl b/iterating_api_endpoint/includes/kafka_stg.incl similarity index 59% rename from iterating_api_endpoint/includes/top_pages_test.incl rename to iterating_api_endpoint/includes/kafka_stg.incl index ed5ceef5..e27294e7 100644 --- a/iterating_api_endpoint/includes/top_pages_test.incl +++ b/iterating_api_endpoint/includes/kafka_stg.incl @@ -1,3 +1,3 @@ NODE include_top_pages SQL > - SELECT 1 + SELECT 1 as stg From 2f3edafc83bdc947bdfe0ce7337e55911f3c2457 Mon Sep 17 00:00:00 2001 From: alrocar Date: Wed, 31 Jul 2024 10:35:50 +0200 Subject: [PATCH 5/9] update --- .../iterating_api_endpoint_ci_prod.yml | 29 ------------------- .../includes/kafka_stg.incl | 11 +++++-- 2 files changed, 8 insertions(+), 32 deletions(-) delete mode 100644 .github/workflows/iterating_api_endpoint_ci_prod.yml diff --git a/.github/workflows/iterating_api_endpoint_ci_prod.yml b/.github/workflows/iterating_api_endpoint_ci_prod.yml deleted file mode 100644 index 73ccaa9e..00000000 --- a/.github/workflows/iterating_api_endpoint_ci_prod.yml +++ /dev/null @@ -1,29 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - branches: - - main - - master - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - paths: - - 'iterating_api_endpoint/**' - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: # ci using environments from workspace 'iterating_api_endpoint' - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.1.0 - with: - data_project_dir: iterating_api_endpoint - tb_env: prod - secrets: - tb_admin_token: ${{ secrets.ITERATING_API_ENDPOINT_TOKEN_PROD }} # set the Workspace admin token in GitHub secrets - tb_host: https://api.tinybird.co diff --git a/iterating_api_endpoint/includes/kafka_stg.incl b/iterating_api_endpoint/includes/kafka_stg.incl index e27294e7..6e0892d5 100644 --- a/iterating_api_endpoint/includes/kafka_stg.incl +++ b/iterating_api_endpoint/includes/kafka_stg.incl @@ -1,3 +1,8 @@ -NODE include_top_pages -SQL > - SELECT 1 as stg +KAFKA_CONNECTION_NAME 'kafka_connection' +KAFKA_TOPIC 'add_column_kafka_ds' +KAFKA_GROUP_ID 'add_column_kafka_ds_ex101' +KAFKA_AUTO_OFFSET_RESET 'earliest' +KAFKA_STORE_RAW_VALUE 'False' +KAFKA_STORE_HEADERS 'False' +KAFKA_TARGET_PARTITIONS 'auto' +KAFKA_KEY_AVRO_DESERIALIZATION '' \ No newline at end of file From 7b8ce9f6a13f530f3aa8c98c319863d443365a9e Mon Sep 17 00:00:00 2001 From: alrocar Date: Wed, 31 Jul 2024 10:36:52 +0200 Subject: [PATCH 6/9] undo --- iterating_api_endpoint/pipes/top_pages.pipe | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/iterating_api_endpoint/pipes/top_pages.pipe b/iterating_api_endpoint/pipes/top_pages.pipe index 88650493..394f362d 100644 --- a/iterating_api_endpoint/pipes/top_pages.pipe +++ b/iterating_api_endpoint/pipes/top_pages.pipe @@ -6,8 +6,6 @@ DESCRIPTION > TOKEN "dashboard" READ -INCLUDE "../includes/top_pages_$TB_ENV.incl" - NODE endpoint DESCRIPTION > Group by pagepath and calcualte hits and visits @@ -18,8 +16,7 @@ SQL > select pathname, uniqMerge(visits) as visits, - countMerge(hits) as hits, - 1 as $TB_ENV + countMerge(hits) as hits from analytics_pages_mv where @@ -34,7 +31,7 @@ SQL > and date <= today() {% end %} group by - pathname, $TB_ENV + pathname order by visits desc limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} From 41b56243acb795a0a6612e3d2926af99fd0c987e Mon Sep 17 00:00:00 2001 From: alrocar Date: Wed, 31 Jul 2024 10:43:58 +0200 Subject: [PATCH 7/9] use env for s3 stg/prod --- .github/workflows/create_s3_ds_cd.yml | 5 +++-- ...nt_cd_prod.yml => create_s3_ds_cd_stg.yml} | 11 ++++------ ...e_s3_ds_ci.yml => create_s3_ds_ci_stg.yml} | 3 ++- .../workflows/iterating_api_endpoint_cd.yml | 3 +-- .../workflows/iterating_api_endpoint_ci.yml | 3 +-- .../datasources/s3_sample.datasource | 6 +---- create_s3_datasource/includes/s3_stg.incl | 5 +++++ create_s3_datasource/includes/stg_prod.incl | 5 +++++ .../datasources/kafka.datasource | 22 ------------------- .../includes/kafka_prod.incl | 8 ------- .../includes/kafka_stg.incl | 8 ------- 11 files changed, 22 insertions(+), 57 deletions(-) rename .github/workflows/{iterating_api_endpoint_cd_prod.yml => create_s3_ds_cd_stg.yml} (61%) rename .github/workflows/{create_s3_ds_ci.yml => create_s3_ds_ci_stg.yml} (90%) create mode 100644 create_s3_datasource/includes/s3_stg.incl create mode 100644 create_s3_datasource/includes/stg_prod.incl delete mode 100644 iterating_api_endpoint/datasources/kafka.datasource delete mode 100644 iterating_api_endpoint/includes/kafka_prod.incl delete mode 100644 iterating_api_endpoint/includes/kafka_stg.incl diff --git a/.github/workflows/create_s3_ds_cd.yml b/.github/workflows/create_s3_ds_cd.yml index eb49eaea..1174703b 100644 --- a/.github/workflows/create_s3_ds_cd.yml +++ b/.github/workflows/create_s3_ds_cd.yml @@ -15,9 +15,10 @@ - main jobs: cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.1 + uses: tinybirdco/ci/.github/workflows/cd.yml@v4.1.0 with: data_project_dir: ./create_s3_datasource + tb_env: prod secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_CREATE_S3_DS }} + tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_CREATE_S3_DS_PROD }} tb_host: https://api.tinybird.co diff --git a/.github/workflows/iterating_api_endpoint_cd_prod.yml b/.github/workflows/create_s3_ds_cd_stg.yml similarity index 61% rename from .github/workflows/iterating_api_endpoint_cd_prod.yml rename to .github/workflows/create_s3_ds_cd_stg.yml index 1c2f701e..881ad82f 100644 --- a/.github/workflows/iterating_api_endpoint_cd_prod.yml +++ b/.github/workflows/create_s3_ds_cd_stg.yml @@ -9,17 +9,14 @@ on: workflow_dispatch: push: - branches: - - main - - master paths: - - 'iterating_api_endpoint/**' + - 'create_s3_datasource/**' jobs: cd: uses: tinybirdco/ci/.github/workflows/cd.yml@v4.1.0 with: - data_project_dir: iterating_api_endpoint - tb_env: prod + data_project_dir: ./create_s3_datasource + tb_env: stg secrets: - tb_admin_token: ${{ secrets.ITERATING_API_ENDPOINT_TOKEN_PROD }} # set the Workspace admin token in GitHub secrets + tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_CREATE_S3_DS }} tb_host: https://api.tinybird.co diff --git a/.github/workflows/create_s3_ds_ci.yml b/.github/workflows/create_s3_ds_ci_stg.yml similarity index 90% rename from .github/workflows/create_s3_ds_ci.yml rename to .github/workflows/create_s3_ds_ci_stg.yml index 623a15de..ec8f7f9c 100644 --- a/.github/workflows/create_s3_ds_ci.yml +++ b/.github/workflows/create_s3_ds_ci_stg.yml @@ -19,9 +19,10 @@ jobs: ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.1 + uses: tinybirdco/ci/.github/workflows/ci.yml@v4.1.0 with: data_project_dir: ./create_s3_datasource + tb_env: stg secrets: tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_CREATE_S3_DS }} tb_host: https://api.tinybird.co diff --git a/.github/workflows/iterating_api_endpoint_cd.yml b/.github/workflows/iterating_api_endpoint_cd.yml index 20a72ce6..bc60e286 100644 --- a/.github/workflows/iterating_api_endpoint_cd.yml +++ b/.github/workflows/iterating_api_endpoint_cd.yml @@ -16,10 +16,9 @@ - 'iterating_api_endpoint/**' jobs: cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.1.0 + uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 with: data_project_dir: iterating_api_endpoint - tb_env: stg secrets: tb_admin_token: ${{ secrets.ITERATING_API_ENDPOINT_TOKEN }} # set the Workspace admin token in GitHub secrets tb_host: https://api.tinybird.co diff --git a/.github/workflows/iterating_api_endpoint_ci.yml b/.github/workflows/iterating_api_endpoint_ci.yml index 9fc27f4e..d78480be 100644 --- a/.github/workflows/iterating_api_endpoint_ci.yml +++ b/.github/workflows/iterating_api_endpoint_ci.yml @@ -20,10 +20,9 @@ jobs: ci: # ci using environments from workspace 'iterating_api_endpoint' - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.1.0 + uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 with: data_project_dir: iterating_api_endpoint - tb_env: stg secrets: tb_admin_token: ${{ secrets.ITERATING_API_ENDPOINT_TOKEN }} # set the Workspace admin token in GitHub secrets tb_host: https://api.tinybird.co diff --git a/create_s3_datasource/datasources/s3_sample.datasource b/create_s3_datasource/datasources/s3_sample.datasource index 3b56e73e..0ae6cfc9 100644 --- a/create_s3_datasource/datasources/s3_sample.datasource +++ b/create_s3_datasource/datasources/s3_sample.datasource @@ -4,8 +4,4 @@ SCHEMA > ENGINE MergeTree ENGINE_SORTING_KEY t -IMPORT_SERVICE s3_iamrole -IMPORT_CONNECTION_NAME create-s3-datasource -IMPORT_BUCKET_URI s3://dev-alrocar/folder1/*.csv -IMPORT_STRATEGY append -IMPORT_SCHEDULE @on-demand +INCLUDE "../includes/s3_$TB_ENV.incl" diff --git a/create_s3_datasource/includes/s3_stg.incl b/create_s3_datasource/includes/s3_stg.incl new file mode 100644 index 00000000..9645c66a --- /dev/null +++ b/create_s3_datasource/includes/s3_stg.incl @@ -0,0 +1,5 @@ +IMPORT_SERVICE s3_iamrole +IMPORT_CONNECTION_NAME create-s3-datasource +IMPORT_BUCKET_URI s3://dev-alrocar/folder1/*.csv +IMPORT_STRATEGY append +IMPORT_SCHEDULE @on-demand \ No newline at end of file diff --git a/create_s3_datasource/includes/stg_prod.incl b/create_s3_datasource/includes/stg_prod.incl new file mode 100644 index 00000000..7b1774da --- /dev/null +++ b/create_s3_datasource/includes/stg_prod.incl @@ -0,0 +1,5 @@ +IMPORT_SERVICE s3_iamrole +IMPORT_CONNECTION_NAME create-s3-datasource +IMPORT_BUCKET_URI s3://dev-alrocar/folder2/*.csv +IMPORT_STRATEGY append +IMPORT_SCHEDULE @on-demand \ No newline at end of file diff --git a/iterating_api_endpoint/datasources/kafka.datasource b/iterating_api_endpoint/datasources/kafka.datasource deleted file mode 100644 index 371507c3..00000000 --- a/iterating_api_endpoint/datasources/kafka.datasource +++ /dev/null @@ -1,22 +0,0 @@ -SCHEMA > - `datetime` DateTime `json:$.datetime`, - `device` String `json:$.device`, - `event_id` String `json:$.event_id`, - `event_name` String `json:$.event_name`, - `fingerprint` String `json:$.fingerprint`, - `price` Int16 `json:$.price`, - `product_id` String `json:$.product_id`, - `product_type` String `json:$.product_type`, - `url` String `json:$.url`, - `user_agent` String `json:$.user_agent`, - `meta_color` Nullable(String) `json:$.meta.color`, - `meta_size` Nullable(String) `json:$.meta.size`, - `meta_image` Nullable(String) `json:$.meta.image`, - `meta_image_v2` Nullable(String) `json:$.meta.image_v2`, - `meta_image_v3` Nullable(String) `json:$.meta.image_v3` - -ENGINE "MergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(__timestamp)" -ENGINE_SORTING_KEY "__timestamp" - -INCLUDE "../includes/kafka_$TB_ENV.incl" \ No newline at end of file diff --git a/iterating_api_endpoint/includes/kafka_prod.incl b/iterating_api_endpoint/includes/kafka_prod.incl deleted file mode 100644 index 954dc70d..00000000 --- a/iterating_api_endpoint/includes/kafka_prod.incl +++ /dev/null @@ -1,8 +0,0 @@ -KAFKA_CONNECTION_NAME 'kafka_connection_prod' -KAFKA_TOPIC 'add_column_kafka_ds_prod' -KAFKA_GROUP_ID 'add_column_kafka_ds_ex101_prod' -KAFKA_AUTO_OFFSET_RESET 'earliest' -KAFKA_STORE_RAW_VALUE 'False' -KAFKA_STORE_HEADERS 'False' -KAFKA_TARGET_PARTITIONS 'auto' -KAFKA_KEY_AVRO_DESERIALIZATION '' \ No newline at end of file diff --git a/iterating_api_endpoint/includes/kafka_stg.incl b/iterating_api_endpoint/includes/kafka_stg.incl deleted file mode 100644 index 6e0892d5..00000000 --- a/iterating_api_endpoint/includes/kafka_stg.incl +++ /dev/null @@ -1,8 +0,0 @@ -KAFKA_CONNECTION_NAME 'kafka_connection' -KAFKA_TOPIC 'add_column_kafka_ds' -KAFKA_GROUP_ID 'add_column_kafka_ds_ex101' -KAFKA_AUTO_OFFSET_RESET 'earliest' -KAFKA_STORE_RAW_VALUE 'False' -KAFKA_STORE_HEADERS 'False' -KAFKA_TARGET_PARTITIONS 'auto' -KAFKA_KEY_AVRO_DESERIALIZATION '' \ No newline at end of file From 477a2ffea2ad36dcf3de8cf84644c325943a634c Mon Sep 17 00:00:00 2001 From: alrocar Date: Wed, 31 Jul 2024 10:46:37 +0200 Subject: [PATCH 8/9] fix --- .github/workflows/create_s3_ds_cd_stg.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/create_s3_ds_cd_stg.yml b/.github/workflows/create_s3_ds_cd_stg.yml index 881ad82f..b939d0cd 100644 --- a/.github/workflows/create_s3_ds_cd_stg.yml +++ b/.github/workflows/create_s3_ds_cd_stg.yml @@ -11,6 +11,8 @@ push: paths: - 'create_s3_datasource/**' + branches: + - main jobs: cd: uses: tinybirdco/ci/.github/workflows/cd.yml@v4.1.0 From 529e926eb1a8b9d8c3c315e9d421ebe188de062c Mon Sep 17 00:00:00 2001 From: alrocar Date: Wed, 31 Jul 2024 12:49:18 +0200 Subject: [PATCH 9/9] Update s3_sample.datasource --- create_s3_datasource/datasources/s3_sample.datasource | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_s3_datasource/datasources/s3_sample.datasource b/create_s3_datasource/datasources/s3_sample.datasource index 0ae6cfc9..d9518c52 100644 --- a/create_s3_datasource/datasources/s3_sample.datasource +++ b/create_s3_datasource/datasources/s3_sample.datasource @@ -4,4 +4,4 @@ SCHEMA > ENGINE MergeTree ENGINE_SORTING_KEY t -INCLUDE "../includes/s3_$TB_ENV.incl" +INCLUDE "../includes/s3_${TB_ENV}.incl"