Skip to content

Commit 464f419

Browse files
kameshsampathkameshsampath
committed
(feat!): end to end working
1 parent 077ffad commit 464f419

23 files changed

+1460
-106
lines changed

‎.dockerignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
**

‎.github/workflows/polaris.yml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
name: Build Apache Polaris
2+
3+
on:
4+
push:
5+
branches: [main]
6+
tags: ["v*.*.*"]
7+
paths-ignore:
8+
- README.md
9+
- LICENSE
10+
- .gitignore
11+
- .tools-version
12+
- docker-compose.yml
13+
pull_request:
14+
branches: [main]
15+
workflow_dispatch:
16+
17+
env:
18+
REGISTRY: ghcr.io
19+
20+
jobs:
21+
base-build-and-push:
22+
runs-on: ubuntu-latest
23+
outputs:
24+
tags: ${{ steps.tags.outputs.tags }}
25+
26+
permissions:
27+
contents: read
28+
packages: write
29+
30+
steps:
31+
- name: Checkout repository
32+
uses: actions/checkout@v4
33+
34+
- name: Set up QEMU
35+
uses: docker/setup-qemu-action@v3
36+
37+
- name: Set up Docker Buildx
38+
uses: docker/setup-buildx-action@v3
39+
40+
- name: Log into GitHub Container Registry
41+
uses: docker/login-action@v3
42+
with:
43+
registry: ${{ env.REGISTRY }}
44+
username: ${{ github.actor }}
45+
password: ${{ secrets.GITHUB_TOKEN }}
46+
47+
- name: Set lowercase image name
48+
run: |
49+
echo "IMAGE_NAME=$(echo ${{ github.repository }}/polaris | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
50+
51+
- name: Extract Docker metadata
52+
id: meta
53+
uses: docker/metadata-action@v5
54+
with:
55+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
56+
tags: |
57+
type=ref,event=branch
58+
type=ref,event=pr
59+
type=semver,pattern={{version}}
60+
type=semver,pattern={{major}}.{{minor}}
61+
type=sha
62+
63+
- name: Generate tags
64+
id: tags
65+
run: |
66+
if [[ "${{ github.ref }}" == "refs/tags/"* ]]; then
67+
VERSION=${GITHUB_REF#refs/tags/v}
68+
echo "tags=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.tag-suffix }},${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.tag-suffix }}-${VERSION}" >> $GITHUB_OUTPUT
69+
else
70+
echo "tags=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.tag-suffix }}" >> $GITHUB_OUTPUT
71+
fi
72+
73+
- name: Build and push Docker image
74+
uses: docker/build-push-action@v5
75+
with:
76+
context: .
77+
dockerfile: Dockerfile.polaris
78+
platforms: linux/amd64,linux/arm64
79+
push: ${{ github.event_name != 'pull_request' }}
80+
tags: ${{ steps.tags.outputs.tags }}
81+
labels: ${{ steps.meta.outputs.labels }}
82+
cache-from: type=gha
83+
cache-to: type=gha,mode=max

‎.github/workflows/pysparknotebook.yml‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
name: Build Apache Spark with Jupyter Notebook
2+
3+
on:
4+
push:
5+
branches: [main]
6+
tags: ["v*.*.*"]
7+
paths-ignore:
8+
- README.md
9+
- LICENSE
10+
- .gitignore
11+
- .tools-version
12+
- docker-compose.yml
13+
pull_request:
14+
branches: [main]
15+
workflow_dispatch:
16+
17+
env:
18+
REGISTRY: ghcr.io
19+
JAVA_VERSION: 17
20+
SPARK_VERSION: 3.5.4
21+
HADOOP_VERSION: 3
22+
jobs:
23+
build-and-push:
24+
runs-on: ubuntu-latest
25+
outputs:
26+
tags: ${{ steps.tags.outputs.tags }}
27+
28+
permissions:
29+
contents: read
30+
packages: write
31+
32+
steps:
33+
- name: Checkout repository
34+
uses: actions/checkout@v4
35+
36+
- name: Set up QEMU
37+
uses: docker/setup-qemu-action@v3
38+
39+
- name: Set up Docker Buildx
40+
uses: docker/setup-buildx-action@v3
41+
42+
- name: Log into GitHub Container Registry
43+
uses: docker/login-action@v3
44+
with:
45+
registry: ${{ env.REGISTRY }}
46+
username: ${{ github.actor }}
47+
password: ${{ secrets.GITHUB_TOKEN }}
48+
49+
- name: Set lowercase image name
50+
run: |
51+
echo "IMAGE_NAME=$(echo ${{ github.repository }}/spark35notebook | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
52+
53+
- name: Extract Docker metadata
54+
id: meta
55+
uses: docker/metadata-action@v5
56+
with:
57+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
58+
tags: |
59+
type=ref,event=branch
60+
type=ref,event=pr
61+
type=semver,pattern={{version}}
62+
type=semver,pattern={{major}}.{{minor}}
63+
type=sha
64+
65+
- name: Generate tags
66+
id: tags
67+
run: |
68+
if [[ "${{ github.ref }}" == "refs/tags/"* ]]; then
69+
VERSION=${GITHUB_REF#refs/tags/v}
70+
echo "tags=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.tag-suffix }},${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.tag-suffix }}-${VERSION}" >> $GITHUB_OUTPUT
71+
else
72+
echo "tags=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.tag-suffix }}" >> $GITHUB_OUTPUT
73+
fi
74+
75+
- name: Build and push Docker image
76+
uses: docker/build-push-action@v5
77+
with:
78+
context: pyspark-notebook
79+
platforms: linux/amd64,linux/arm64
80+
push: ${{ github.event_name != 'pull_request' }}
81+
tags: ${{ steps.tags.outputs.tags }}
82+
labels: ${{ steps.meta.outputs.labels }}
83+
build-args: |
84+
spark_version=${{ env.SPARK_VERSION }}
85+
openjdk_version=${{ env.JAVA_VERSION }}
86+
hadoop_version=${{ env.HADOOP_VERSION }}
87+
cache-from: type=gha
88+
cache-to: type=gha,mode=max

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
work

‎Dockerfile.polaris‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
FROM gitscm/git AS src
2+
ARG POLARIS_VERSION=0.9.x
3+
4+
# clone the Polaris repository
5+
RUN git clone -b ${POLARIS_VERSION} https://github.com/apache/polaris.git
6+
7+
FROM registry.access.redhat.com/ubi9/openjdk-21:1.20-2.1726695192 AS build
8+
ARG ECLIPSELINK=false
9+
ARG ECLIPSELINK_DEPS
10+
11+
# Copy the REST catalog into the container
12+
COPY --from=src --chown=default:root polaris/ /app
13+
14+
# Set the working directory in the container, nuke any existing builds
15+
WORKDIR /app
16+
RUN rm -rf build
17+
18+
# Build the rest catalog
19+
RUN ./gradlew --no-daemon --info ${ECLIPSELINK_DEPS+"-PeclipseLinkDeps=$ECLIPSELINK_DEPS"} -PeclipseLink=$ECLIPSELINK clean prepareDockerDist
20+
21+
FROM registry.access.redhat.com/ubi9/openjdk-21-runtime:1.20-2.1729089285
22+
23+
WORKDIR /app
24+
25+
COPY --from=build /app/dropwizard/service/build/docker-dist/bin /app/bin
26+
COPY --from=build /app/dropwizard/service/build/docker-dist/lib /app/lib
27+
COPY --from=build /app/polaris-server.yml /app
28+
29+
USER root
30+
31+
# directory for the Iceberg and Catalog files that will use storage type as FILE
32+
# more for demo and testing purposes
33+
RUN mkdir -p /data && \
34+
# Set root group ownership
35+
chown root:root /data && \
36+
# Set SGID bit and group write permissions
37+
chmod 2775 /data && \
38+
sed -i '/CLASSPATH=.*/{p;s/.*/CLASSPATH="$CLASSPATH:\/conf"/}' /app/bin/polaris-service
39+
40+
USER default
41+
42+
EXPOSE 8181
43+
44+
# Run the resulting java binary
45+
ENTRYPOINT ["/app/bin/polaris-service"]
46+
CMD ["server", "polaris-server.yml"]

0 commit comments

Comments
 (0)