Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,6 @@ demo/jars/*
demo/notebook/.ipynb_checkpoints/*
my_config.yaml
my_config_catalog.yaml

# REST generated models
spec/generated
72 changes: 72 additions & 0 deletions spec/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements…
# (your license header…)

PYTHON := python3
VENV_DIR := .venv
PIP := $(VENV_DIR)/bin/pip
VALIDATOR := $(VENV_DIR)/bin/openapi-spec-validator
SPEC := rest-service-open-api.yaml

# Configuration for model generation
GEN_DIR := generated
MODEL_PKG := org.apache.xtable.service.models

.PHONY: all venv install lint clean clean-models

all: lint

# Create venv if it doesn't exist
venv:
@echo "→ creating virtualenv in $(VENV_DIR)…"
$(PYTHON) -m venv $(VENV_DIR)
@echo "→ upgrading pip…"
$(PIP) install --upgrade pip

# Install requirements into venv
install: venv
@echo "→ installing validator…"
$(PIP) install -r requirements.txt

# Validate your OpenAPI spec
lint: install
@echo "→ linting $(SPEC)…"
$(VALIDATOR) --errors all $(SPEC)

# Remove the virtualenv
clean:
@echo "→ removing $(VENV_DIR)…"
rm -rf $(VENV_DIR)

# Generate Java model classes from OpenAPI spec
generate-models:
@echo "→ generating Java model classes from $(SPEC)…"
openapi-generator generate \
-i $(SPEC) \
-g java \
-o $(GEN_DIR) \
--model-package $(MODEL_PKG) \
--global-property models,modelTests=false,modelDocs=false
@echo "→ models generated in $(GEN_DIR)/src/main/java/$(subst .,/,$(MODEL_PKG))"

# Remove all generated model classes
clean-models:
@echo "→ removing generated models in $(GEN_DIR)…"
rm -rf $(GEN_DIR)
53 changes: 53 additions & 0 deletions spec/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-->

# Open API spec

The `rest-service-open-api.yaml` defines the api contract for running table format conversion using XTable's REST service.
This spec is still under active development and is subject to changes.

## Lint

To make sure that the open-api definition is valid, you can run the `lint` command:

This will first run `make install`, which in turn will create the venv (if needed) and install openapi-spec-validator
```sh
make lint
```

if you want to wipe out the venv and start fresh run make clean
```sh
make clean
```

## Generate

Note: You’ll need the OpenAPI Generator CLI installed and on your `PATH`. You can install it using Homebrew:
```sh
brew install openapi-generator
```
Then to generate the java models from the open-api spec, you can run the `generate-models` command.

```sh
make generate-models
```
If you would to remove the generated models, you can run the `clean-models` command:
```sh
make clean-models
```
18 changes: 18 additions & 0 deletions spec/requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

openapi-spec-validator==0.7.1
240 changes: 240 additions & 0 deletions spec/rest-service-open-api.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,240 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
openapi: 3.0.3
info:
title: XTable REST Service API
license:
name: Apache 2.0
url: https://www.apache.org/licenses/LICENSE-2.0.html
version: 0.0.1
description: |
A REST API for initiating metadata conversion using Apache XTable. Note this spec is still under active development and is subject to changes.

servers:
- url: "{scheme}://{host}/{prefix}"
description: Server URL when the port can be inferred from the scheme
variables:
scheme:
description: The scheme of the URI, either http or https.
default: https
host:
description: The host address for the specified server
default: localhost
prefix:
description: Optional prefix to be appended to all routes
default: ""
- url: "{scheme}://{host}:{port}/{prefix}"
description: Generic base server URL, with all parts configurable
variables:
scheme:
description: The scheme of the URI, either http or https.
default: https
host:
description: The host address for the specified server
default: localhost
port:
description: The port used when addressing the host
default: "8080"
prefix:
description: Optional prefix to be appended to all routes
default: ""

paths:
/v1/conversion/table:
post:
tags:
- XTable Service API
summary: Initiate XTable's runSync process to convert a source table format to the requested target table formats.
description: |
Reads the source table metadata from storage, converts it to the requested
target formats.
operationId: convertTable
parameters:
- in: header
name: Prefer
description: Use 'respond-async' to request asynchronous processing.
required: false
schema:
type: string
enum:
- respond-async
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/ConvertTableRequest'
responses:
'200':
$ref: '#/components/responses/ConvertTableResponse'
'202':
$ref: '#/components/responses/SubmittedConversionResponse'
'403':
$ref: '#/components/responses/ForbiddenResponse'
'503':
$ref: '#/components/responses/ServiceUnavailableResponse'
default:
$ref: '#/components/responses/ErrorResponse'

/v1/conversion/table/{conversion-id}:
get:
tags:
- XTable Service API
summary: Polls on a conversion-id to see if converting a table has finished.
operationId: getConversionStatus
parameters:
- in: path
name: conversion-id
required: true
schema:
type: string
responses:
'200':
$ref: '#/components/responses/ConvertTableResponse'
'202':
description: Still processing; conversion has not yet finished.
'403':
$ref: '#/components/responses/ForbiddenResponse'
'503':
$ref: '#/components/responses/ServiceUnavailableResponse'
default:
$ref: '#/components/responses/ErrorResponse'

components:
schemas:
ConvertTableRequest:
type: object
required:
- source-format
- source-table-name
- source-table-path
- target-formats
properties:
source-format:
type: string
description: Name of the source table format (e.g., "ICEBERG", "HUDI", "DELTA")
source-table-name:
type: string
description: Name of the source table
source-table-path:
type: string
description: Path to the source table's metadata
target-formats:
type: array
items:
type: string
description: List of desired output table formats (e.g., "ICEBERG", "HUDI", "DELTA")
configurations:
type: object
description: Additional configuration key/value pairs (e.g., storage credentials or other service configs)
additionalProperties:
type: string

TargetTable:
type: object
required:
- target-format
- target-metadata-path
properties:
target-format:
type: string
description: Name of the target format (e.g., "ICEBERG", "HUDI", "DELTA")
target-metadata-path:
type: string
description: Path where the converted metadata was written
target-schema:
type: string
description: Schema definition of the converted table

ConvertTableResponse:
type: object
required:
- conversions
properties:
conversions:
type: array
description: A list of converted tables, one per requested format
items:
$ref: '#/components/schemas/TargetTable'

SubmittedConversionResponse:
type: object
properties:
conversion-id:
type: string
description: ID to use when polling for the final result

ErrorModel:
type: object
description: JSON error payload returned in a response with further details on the error
required:
- message
- type
- code
properties:
message:
type: string
description: Human-readable error message
type:
type: string
description: Internal type definition of the error
code:
type: integer
minimum: 400
maximum: 600
description: HTTP response code
stack:
type: array
description: Optional stack trace elements for debugging
items:
type: string

responses:
ConvertTableResponse:
description: Successful conversion result
content:
application/json:
schema:
$ref: '#/components/schemas/ConvertTableResponse'

SubmittedConversionResponse:
description: Request accepted for asynchronous processing
content:
application/json:
schema:
$ref: '#/components/schemas/SubmittedConversionResponse'

ForbiddenResponse:
description: Caller does not have permission to perform this operation
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorModel'

ServiceUnavailableResponse:
description: Conversion service is temporarily unavailable
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorModel'

ErrorResponse:
description: Unexpected error
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorModel'