diff --git a/services/tables/build.gradle b/services/tables/build.gradle index daadd8297..ec65b9fd3 100644 --- a/services/tables/build.gradle +++ b/services/tables/build.gradle @@ -30,6 +30,92 @@ openApi { apiDocsUrl.set("http://localhost:8000/v3/api-docs") } +def canonicalIcebergRestSpec = "${rootDir}/spec/iceberg-rest-catalog-open-api.yaml" +def icebergRestSpecVersion = "apache-iceberg-1.10.0" +def icebergRestSpecUrl = "https://raw.githubusercontent.com/apache/iceberg/refs/tags/${icebergRestSpecVersion}/open-api/rest-catalog-open-api.yaml" +def openApiCliVersion = "7.12.0" +def openApiCliDir = "${buildDir}/openapi-cli" +def openApiCliJar = "${openApiCliDir}/openapi-generator-cli-${openApiCliVersion}.jar" +def generatedIcebergRestDir = "${buildDir}/generated/openapi/iceberg-rest-server" + +/** + * The Iceberg library has its own hand-written classes (ConfigResponse, LoadTableResponse, etc.) + * that predate the REST OpenAPI spec. These library classes use builders, custom Jackson + * serializers, and kebab-case — they don't follow JavaBean conventions. The OpenAPI codegen + * would normally generate a separate set of standard POJO model classes for the same types. + * + * Rather than maintaining adapters between two representations of the same wire format, we tell + * the codegen to skip model generation (models=false) and reference the Iceberg library classes + * directly in the generated interfaces. This is the same approach used by Apache Polaris + * (see polaris/api/iceberg-service/build.gradle.kts). + * + * Every schema in the spec needs an explicit mapping, otherwise the generated code references + * a class that was never generated. Types added in Iceberg 1.10+ that don't exist in our 1.5.2 + * fork are mapped to Object — the generated default-501 methods need something to compile against. + * These Object mappings can be removed when OpenHouse upgrades to Iceberg 1.7+. + */ +// importMappings controls the `import` statement in generated files. +// typeMappings controls the inline type name in method signatures and @Schema annotations. +// Both are needed due to an openapi-generator bug — setting one does not update the other. +def icebergImportMappings = [ + // Iceberg REST responses + "CatalogConfig=org.apache.iceberg.rest.responses.ConfigResponse", + "CommitTableResponse=org.apache.iceberg.rest.responses.LoadTableResponse", + "CreateNamespaceResponse=org.apache.iceberg.rest.responses.CreateNamespaceResponse", + "ErrorModel=org.apache.iceberg.rest.responses.ErrorResponse", + "GetNamespaceResponse=org.apache.iceberg.rest.responses.GetNamespaceResponse", + "IcebergErrorResponse=org.apache.iceberg.rest.responses.ErrorResponse", + "ListNamespacesResponse=org.apache.iceberg.rest.responses.ListNamespacesResponse", + "ListTablesResponse=org.apache.iceberg.rest.responses.ListTablesResponse", + "LoadTableResult=org.apache.iceberg.rest.responses.LoadTableResponse", + "LoadViewResult=org.apache.iceberg.rest.responses.LoadTableResponse", + "OAuthError=org.apache.iceberg.rest.responses.ErrorResponse", + "OAuthErrorResponse=org.apache.iceberg.rest.responses.OAuthErrorResponse", + "OAuthTokenResponse=org.apache.iceberg.rest.responses.OAuthTokenResponse", + "UpdateNamespacePropertiesResponse=org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse", + // Iceberg REST requests + "CreateNamespaceRequest=org.apache.iceberg.rest.requests.CreateNamespaceRequest", + "CreateTableRequest=org.apache.iceberg.rest.requests.CreateTableRequest", + "CreateViewRequest=org.apache.iceberg.rest.requests.CreateViewRequest", + "CommitTransactionRequest=org.apache.iceberg.rest.requests.CommitTransactionRequest", + "RegisterTableRequest=org.apache.iceberg.rest.requests.RegisterTableRequest", + "RenameTableRequest=org.apache.iceberg.rest.requests.RenameTableRequest", + "ReportMetricsRequest=org.apache.iceberg.rest.requests.ReportMetricsRequest", + "UpdateNamespacePropertiesRequest=org.apache.iceberg.rest.requests.UpdateNamespacePropertiesRequest", + // Types not in Iceberg 1.5.2 — mapped to Object so unimplemented default methods compile + "CommitTableRequest=java.lang.Object", + "CommitViewRequest=java.lang.Object", + "FetchPlanningResult=java.lang.Object", + "FetchScanTasksRequest=java.lang.Object", + "FetchScanTasksResult=java.lang.Object", + "LoadCredentialsResponse=java.lang.Object", + "PlanTableScanRequest=java.lang.Object", + "PlanTableScanResult=java.lang.Object", + // OAuth2 model type suppressed by models=false + "TokenType=java.lang.String", +].join(",") + +def icebergTypeMappings = [ + "CatalogConfig=org.apache.iceberg.rest.responses.ConfigResponse", + "CommitTableResponse=org.apache.iceberg.rest.responses.LoadTableResponse", + "ErrorModel=org.apache.iceberg.rest.responses.ErrorResponse", + "IcebergErrorResponse=org.apache.iceberg.rest.responses.ErrorResponse", + "LoadTableResult=org.apache.iceberg.rest.responses.LoadTableResponse", + "LoadViewResult=org.apache.iceberg.rest.responses.LoadTableResponse", + "OAuthError=org.apache.iceberg.rest.responses.ErrorResponse", + // Types not in Iceberg 1.5.2 — must also be in typeMappings for @Schema annotations + "CommitTableRequest=java.lang.Object", + "CommitViewRequest=java.lang.Object", + "FetchPlanningResult=java.lang.Object", + "FetchScanTasksRequest=java.lang.Object", + "FetchScanTasksResult=java.lang.Object", + "LoadCredentialsResponse=java.lang.Object", + "PlanTableScanRequest=java.lang.Object", + "PlanTableScanResult=java.lang.Object", + "TokenType=java.lang.String", +].join(",") + + dependencies { compileOnly "io.opentelemetry.instrumentation:opentelemetry-instrumentation-annotations:${otel_annotations_version}" api project(':services:common') @@ -41,6 +127,7 @@ dependencies { api 'org.springframework.security:spring-security-config:5.7.2' api 'org.springframework.boot:spring-boot-starter-webflux:2.7.8' implementation 'com.cronutils:cron-utils:9.2.0' + implementation 'org.openapitools:jackson-databind-nullable:0.2.1' testImplementation 'org.junit.jupiter:junit-jupiter-engine:' + junit_version testImplementation 'org.springframework.security:spring-security-test:5.7.3' testImplementation(testFixtures(project(':services:common'))) @@ -48,3 +135,109 @@ dependencies { exclude group: 'com.linkedin.iceberg' } } + +tasks.register('icebergRestDownloadCodegenCli', Exec) { + commandLine "sh", "${project(':client:common').projectDir}/jar_download.sh", + "-o", openApiCliDir, + "-v", openApiCliVersion + outputs.file(openApiCliJar) +} + +tasks.register('icebergRestValidateSpec', Exec) { + dependsOn tasks.named('icebergRestDownloadCodegenCli') + commandLine "java", "-jar", openApiCliJar, "validate", "-i", canonicalIcebergRestSpec + inputs.file(canonicalIcebergRestSpec) + inputs.file(openApiCliJar) +} + +tasks.register('icebergRestCodegen', Exec) { + dependsOn tasks.named('icebergRestValidateSpec') + doFirst { + delete generatedIcebergRestDir + } + commandLine "java", "-jar", openApiCliJar, "generate", + "-g", "spring", + "-i", canonicalIcebergRestSpec, + "-o", generatedIcebergRestDir, + "--api-package", "com.linkedin.openhouse.tables.generated.iceberg.api", + "--model-package", "com.linkedin.openhouse.tables.generated.iceberg.model", + "--import-mappings", icebergImportMappings, + "--type-mappings", icebergTypeMappings, + "--global-property", "apis,models=false,apiTests=false,apiDocs=false,modelTests=false,modelDocs=false,supportingFiles=", + "--additional-properties", "interfaceOnly=true,useTags=true,skipDefaultInterface=false,hideGenerationTimestamp=true,useSpringBoot3=false" + // Post-process generated code for Iceberg 1.5.2 compatibility. + // + // Problem: importMappings/typeMappings handle return types and @Schema annotations, but the + // openapi-generator does NOT apply typeMappings to @RequestBody parameter types when + // models=false. This leaves bare references to types that don't exist in Iceberg 1.5.2 + // (they were added in 1.10+), causing compilation failures. + // + // Fix: regex-replace those bare type names with Object in generated .java files. + // The regex `\bTypeName\b(?=\s+\w)` matches "TypeName paramName" patterns (i.e., a type + // followed by whitespace and a parameter name) without touching import statements, comments, + // or string literals where the type name isn't used as a declaration. + // + // This workaround can be removed when OpenHouse upgrades to Iceberg 1.7+ where these types + // exist natively. Track via: https://github.com/linkedin/openhouse/issues/XXX + doLast { + def replacements = [ + 'CommitTableRequest': 'Object', + 'CommitViewRequest': 'Object', + 'FetchScanTasksRequest': 'Object', + 'PlanTableScanRequest': 'Object', + ] + fileTree(generatedIcebergRestDir).matching { include '**/*.java' }.each { File f -> + def text = f.text + def original = text + replacements.each { from, to -> + text = text.replaceAll("\\b${from}\\b(?=\\s+\\w)", to) + } + if (text != original) { + f.text = text + } + } + } + inputs.file(canonicalIcebergRestSpec) + inputs.file(openApiCliJar) + outputs.dir(generatedIcebergRestDir) +} + +tasks.named('compileJava') { + dependsOn tasks.named('icebergRestCodegen') + source "${generatedIcebergRestDir}/src/main/java" +} + +tasks.register('icebergRestDownloadUpstreamSpec') { + description = 'Download upstream Iceberg REST spec for sync verification' + def upstreamFile = new File("${buildDir}/iceberg-rest-spec-upstream-${icebergRestSpecVersion}.yaml") + outputs.file(upstreamFile) + onlyIf { !upstreamFile.exists() } + doLast { + upstreamFile.parentFile.mkdirs() + new URL(icebergRestSpecUrl).withInputStream { is -> upstreamFile.bytes = is.bytes } + } +} + +tasks.register('icebergRestVerifySpecSync') { + description = 'Verify vendored Iceberg REST spec matches upstream tag' + dependsOn tasks.named('icebergRestDownloadUpstreamSpec') + def upstreamFile = new File("${buildDir}/iceberg-rest-spec-upstream-${icebergRestSpecVersion}.yaml") + inputs.file(canonicalIcebergRestSpec) + inputs.file(upstreamFile) + outputs.file("${buildDir}/iceberg-rest-spec-sync-verified") + doLast { + def vendored = file(canonicalIcebergRestSpec).text + def upstream = upstreamFile.text + if (vendored != upstream) { + throw new GradleException( + "Vendored spec ${canonicalIcebergRestSpec} does not match upstream ${icebergRestSpecUrl}. " + + "Run: curl -o spec/iceberg-rest-catalog-open-api.yaml ${icebergRestSpecUrl}") + } + file("${buildDir}/iceberg-rest-spec-sync-verified").text = "OK" + } +} + +tasks.named('check') { + dependsOn tasks.named('icebergRestValidateSpec') + dependsOn tasks.named('icebergRestVerifySpecSync') +} diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestCatalogController.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestCatalogController.java new file mode 100644 index 000000000..d58e2253f --- /dev/null +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestCatalogController.java @@ -0,0 +1,140 @@ +package com.linkedin.openhouse.tables.controller; + +import static com.linkedin.openhouse.common.security.AuthenticationUtils.extractAuthenticatedUserPrincipal; + +import com.linkedin.openhouse.common.exception.NoSuchUserTableException; +import com.linkedin.openhouse.internal.catalog.OpenHouseInternalCatalog; +import com.linkedin.openhouse.tables.api.validator.TablesApiValidator; +import com.linkedin.openhouse.tables.generated.iceberg.api.CatalogApiApi; +import com.linkedin.openhouse.tables.generated.iceberg.api.ConfigurationApiApi; +import com.linkedin.openhouse.tables.services.TablesService; +import io.swagger.v3.oas.annotations.Hidden; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.rest.CatalogHandlers; +import org.apache.iceberg.rest.RESTUtil; +import org.apache.iceberg.rest.responses.ConfigResponse; +import org.apache.iceberg.rest.responses.ListTablesResponse; +import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.context.request.NativeWebRequest; + +/** + * Read-only Iceberg REST Catalog surface. + * + *

Implements the generated {@link CatalogApiApi} and {@link ConfigurationApiApi} interfaces from + * the upstream Iceberg REST OpenAPI spec. Only config/list/load/exists are overridden; all other + * endpoints inherit the generated 501 (Not Implemented) default. + * + *

The {@code /v1/config} endpoint returns a {@code prefix} override so that the Iceberg REST + * client addresses all subsequent requests via {@code /v1/{prefix}/namespaces/...}, keeping them + * separate from the existing OpenHouse API routes under {@code /v1/databases/...}. + * + *

Serialization of Iceberg REST types is handled by {@link IcebergRestHttpMessageConverter}, + * registered in {@link IcebergRestSerdeConfig}. + */ +@Hidden // Exclude from SpringDoc OpenAPI spec to avoid operationId clashes with OpenHouse API +@RestController +public class IcebergRestCatalogController implements CatalogApiApi, ConfigurationApiApi { + + /** Prefix returned by {@code /v1/config} and used in all Iceberg REST routes. */ + public static final String ICEBERG_REST_PREFIX = "iceberg"; + + private final OpenHouseInternalCatalog openHouseInternalCatalog; + + private final TablesService tablesService; + + private final TablesApiValidator tablesApiValidator; + + public IcebergRestCatalogController( + OpenHouseInternalCatalog openHouseInternalCatalog, + TablesService tablesService, + TablesApiValidator tablesApiValidator) { + this.openHouseInternalCatalog = openHouseInternalCatalog; + this.tablesService = tablesService; + this.tablesApiValidator = tablesApiValidator; + } + + /** Resolves the diamond-inherited {@code getRequest()} from both interfaces. */ + @Override + public Optional getRequest() { + return Optional.empty(); + } + + @Override + public ResponseEntity getConfig(String warehouse) { + ConfigResponse response = + ConfigResponse.builder().withOverride("prefix", ICEBERG_REST_PREFIX).build(); + return ResponseEntity.ok(response); + } + + @Override + public ResponseEntity listTables( + String prefix, String namespace, String pageToken, Integer pageSize) { + Namespace icebergNamespace = decodeSingleLevelNamespace(namespace); + String databaseId = icebergNamespace.level(0); + tablesApiValidator.validateSearchTables(databaseId); + + List tableIdentifiers = + tablesService.searchTables(databaseId, extractAuthenticatedUserPrincipal()).stream() + .map(table -> TableIdentifier.of(icebergNamespace, table.getTableId())) + .collect(Collectors.toList()); + ListTablesResponse response = ListTablesResponse.builder().addAll(tableIdentifiers).build(); + return ResponseEntity.ok(response); + } + + @Override + public ResponseEntity loadTable( + String prefix, + String namespace, + String table, + String xIcebergAccessDelegation, + String ifNoneMatch, + String snapshots) { + Namespace icebergNamespace = decodeSingleLevelNamespace(namespace); + String databaseId = icebergNamespace.level(0); + tablesApiValidator.validateGetTable(databaseId, table); + + // Reuse the existing table-read authorization and lock visibility checks. + try { + tablesService.getTable(databaseId, table, extractAuthenticatedUserPrincipal()); + } catch (NoSuchUserTableException e) { + throw new NoSuchTableException("Table does not exist: %s.%s", databaseId, table); + } + + LoadTableResponse response = + CatalogHandlers.loadTable( + openHouseInternalCatalog, TableIdentifier.of(icebergNamespace, table)); + return ResponseEntity.ok(response); + } + + @Override + public ResponseEntity tableExists(String prefix, String namespace, String table) { + Namespace icebergNamespace = decodeSingleLevelNamespace(namespace); + String databaseId = icebergNamespace.level(0); + tablesApiValidator.validateGetTable(databaseId, table); + + try { + tablesService.getTable(databaseId, table, extractAuthenticatedUserPrincipal()); + } catch (NoSuchUserTableException e) { + throw new NoSuchTableException("Table does not exist: %s.%s", databaseId, table); + } + + return ResponseEntity.noContent().build(); + } + + private Namespace decodeSingleLevelNamespace(String encodedNamespace) { + Namespace namespace = RESTUtil.decodeNamespace(encodedNamespace); + if (namespace.isEmpty() || namespace.levels().length != 1) { + throw new NoSuchNamespaceException("Invalid namespace: %s", namespace); + } + + return namespace; + } +} diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestExceptionHandler.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestExceptionHandler.java new file mode 100644 index 000000000..0d56a9a88 --- /dev/null +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestExceptionHandler.java @@ -0,0 +1,64 @@ +package com.linkedin.openhouse.tables.controller; + +import com.linkedin.openhouse.common.exception.RequestValidationFailureException; +import org.apache.iceberg.exceptions.ForbiddenException; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.rest.responses.ErrorResponse; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.AccessDeniedException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +/** Scoped exception mapper for Iceberg REST endpoints. */ +@Order(Ordered.HIGHEST_PRECEDENCE) +@RestControllerAdvice(assignableTypes = IcebergRestCatalogController.class) +public class IcebergRestExceptionHandler { + + @ExceptionHandler(NoSuchTableException.class) + public ResponseEntity handleNoSuchTable(NoSuchTableException e) { + return errorResponse(404, e.getMessage(), NoSuchTableException.class.getSimpleName(), e); + } + + @ExceptionHandler(NoSuchNamespaceException.class) + public ResponseEntity handleNoSuchNamespace(NoSuchNamespaceException e) { + return errorResponse(404, e.getMessage(), NoSuchNamespaceException.class.getSimpleName(), e); + } + + @ExceptionHandler({RequestValidationFailureException.class, IllegalArgumentException.class}) + public ResponseEntity handleBadRequest(Exception e) { + return errorResponse(400, e.getMessage(), IllegalArgumentException.class.getSimpleName(), e); + } + + @ExceptionHandler(AccessDeniedException.class) + public ResponseEntity handleForbidden(AccessDeniedException e) { + return errorResponse(403, e.getMessage(), ForbiddenException.class.getSimpleName(), e); + } + + @ExceptionHandler(UnsupportedOperationException.class) + public ResponseEntity handleNotImplemented(UnsupportedOperationException e) { + return errorResponse( + 501, e.getMessage(), UnsupportedOperationException.class.getSimpleName(), e); + } + + @ExceptionHandler(Exception.class) + public ResponseEntity handleDefault(Exception e) { + return errorResponse(500, e.getMessage(), e.getClass().getSimpleName(), e); + } + + private ResponseEntity errorResponse( + int statusCode, String message, String type, Throwable throwable) { + ErrorResponse response = + ErrorResponse.builder() + .responseCode(statusCode) + .withMessage(message) + .withType(type) + .build(); + return ResponseEntity.status(statusCode) + .contentType(MediaType.APPLICATION_JSON) + .body(IcebergRestSerde.toJson(response)); + } +} diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestHttpMessageConverter.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestHttpMessageConverter.java new file mode 100644 index 000000000..e55f106fe --- /dev/null +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestHttpMessageConverter.java @@ -0,0 +1,45 @@ +package com.linkedin.openhouse.tables.controller; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import org.apache.iceberg.rest.RESTResponse; +import org.springframework.http.HttpInputMessage; +import org.springframework.http.HttpOutputMessage; +import org.springframework.http.MediaType; +import org.springframework.http.converter.AbstractHttpMessageConverter; + +/** + * Spring {@link org.springframework.http.converter.HttpMessageConverter} for Iceberg {@link + * RESTResponse} types. Uses {@link IcebergRestSerde} (Iceberg custom serializers, kebab-case) to + * write JSON because Iceberg REST types do not follow JavaBean conventions and cannot be serialized + * by Spring's default Jackson converter. + * + *

This converter only handles writes (responses). Deserialization is not supported because we do + * not accept Iceberg REST request bodies through Spring MVC. + */ +public class IcebergRestHttpMessageConverter extends AbstractHttpMessageConverter { + + public IcebergRestHttpMessageConverter() { + super(MediaType.APPLICATION_JSON); + } + + @Override + protected boolean supports(Class clazz) { + return RESTResponse.class.isAssignableFrom(clazz); + } + + @Override + protected RESTResponse readInternal( + Class clazz, HttpInputMessage inputMessage) { + throw new UnsupportedOperationException("Iceberg REST request deserialization not supported"); + } + + @Override + protected void writeInternal(RESTResponse response, HttpOutputMessage outputMessage) + throws IOException { + OutputStream body = outputMessage.getBody(); + body.write(IcebergRestSerde.toJson(response).getBytes(StandardCharsets.UTF_8)); + body.flush(); + } +} diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestSerde.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestSerde.java new file mode 100644 index 000000000..e57b20df4 --- /dev/null +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestSerde.java @@ -0,0 +1,33 @@ +package com.linkedin.openhouse.tables.controller; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import org.apache.iceberg.rest.RESTSerializers; + +/** Serde helper for Iceberg REST payloads that require kebab-case and Iceberg serializers. */ +final class IcebergRestSerde { + + private static final ObjectMapper MAPPER = new ObjectMapper(new JsonFactory()); + + static { + MAPPER.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + MAPPER.setPropertyNamingStrategy(new PropertyNamingStrategies.KebabCaseStrategy()); + RESTSerializers.registerAll(MAPPER); + } + + private IcebergRestSerde() {} + + static String toJson(Object payload) { + try { + return MAPPER.writeValueAsString(payload); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Unable to serialize Iceberg REST response payload", e); + } + } +} diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestSerdeConfig.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestSerdeConfig.java new file mode 100644 index 000000000..ce9cd8b8e --- /dev/null +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestSerdeConfig.java @@ -0,0 +1,20 @@ +package com.linkedin.openhouse.tables.controller; + +import java.util.List; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +/** + * Registers the {@link IcebergRestHttpMessageConverter} so that Spring MVC can serialize typed + * Iceberg REST responses ({@code ConfigResponse}, {@code LoadTableResponse}, etc.) returned by + * {@link IcebergRestCatalogController}. + */ +@Configuration +public class IcebergRestSerdeConfig implements WebMvcConfigurer { + + @Override + public void extendMessageConverters(List> converters) { + converters.add(0, new IcebergRestHttpMessageConverter()); + } +} diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesService.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesService.java index 363ec0cdd..abbe4f5cf 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesService.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesService.java @@ -32,6 +32,15 @@ public interface TablesService { */ List searchTables(String databaseId); + /** + * Given a databaseId, prepare list of {@link TableDto}s if actingPrincipal has list permission. + * + * @param databaseId + * @param actingPrincipal + * @return list of {@link TableDto} + */ + List searchTables(String databaseId, String actingPrincipal); + /** * Given a databaseId, prepare list of {@link TableDto}s. * diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java index 050e6635f..9500e6273 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java @@ -79,6 +79,13 @@ public List searchTables(String databaseId) { return openHouseInternalRepository.searchTables(databaseId); } + @Override + public List searchTables(String databaseId, String actingPrincipal) { + authorizationUtils.checkDatabasePrivilege( + databaseId, actingPrincipal, Privileges.GET_TABLE_METADATA); + return searchTables(databaseId); + } + @Override public Page searchTables(String databaseId, int page, int size, String sortBy) { Pageable pageable = createPageable(page, size, sortBy, null); diff --git a/spec/iceberg-rest-catalog-open-api.yaml b/spec/iceberg-rest-catalog-open-api.yaml new file mode 100644 index 000000000..e0b524a6d --- /dev/null +++ b/spec/iceberg-rest-catalog-open-api.yaml @@ -0,0 +1,4951 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +openapi: 3.1.1 +info: + title: Apache Iceberg REST Catalog API + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0.html + version: 0.0.1 + description: + Defines the specification for the first version of the REST Catalog API. + Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. +servers: + - url: "{scheme}://{host}/{basePath}" + description: Server URL when the port can be inferred from the scheme + variables: + scheme: + description: The scheme of the URI, either http or https. + default: https + host: + description: The host address for the specified server + default: localhost + basePath: + description: Optional prefix to be appended to all routes + default: "" + - url: "{scheme}://{host}:{port}/{basePath}" + description: Generic base server URL, with all parts configurable + variables: + scheme: + description: The scheme of the URI, either http or https. + default: https + host: + description: The host address for the specified server + default: localhost + port: + description: The port used when addressing the host + default: "443" + basePath: + description: Optional prefix to be appended to all routes + default: "" +# All routes are currently configured using an Authorization header. +security: + - OAuth2: [catalog] + - BearerAuth: [] + +paths: + /v1/config: + + get: + tags: + - Configuration API + summary: List all catalog configuration settings + operationId: getConfig + parameters: + - name: warehouse + in: query + required: false + schema: + type: string + description: Warehouse location or identifier to request from the service + description: + " + All REST clients should first call this route to get catalog configuration + properties from the server to configure the catalog and its HTTP client. + Configuration from the server consists of two sets of key/value pairs. + + - defaults - properties that should be used as default configuration; applied before client configuration + + - overrides - properties that should be used to override client configuration; applied after defaults and client configuration + + + Catalog configuration is constructed by setting the defaults, then client- + provided configuration, and finally overrides. The final property set is then + used to configure the catalog. + + + For example, a default configuration property might set the size of the + client pool, which can be replaced with a client-specific setting. An + override might be used to set the warehouse location, which is stored + on the server rather than in client configuration. + + + Common catalog configuration settings are documented at + https://iceberg.apache.org/docs/latest/configuration/#catalog-properties + + + The catalog configuration also holds an optional `endpoints` field that contains information about the endpoints + supported by the server. If a server does not send the `endpoints` field, a default set of endpoints is assumed: + + - GET /v1/{prefix}/namespaces + + - POST /v1/{prefix}/namespaces + + - GET /v1/{prefix}/namespaces/{namespace} + + - DELETE /v1/{prefix}/namespaces/{namespace} + + - POST /v1/{prefix}/namespaces/{namespace}/properties + + - GET /v1/{prefix}/namespaces/{namespace}/tables + + - POST /v1/{prefix}/namespaces/{namespace}/tables + + - GET /v1/{prefix}/namespaces/{namespace}/tables/{table} + + - POST /v1/{prefix}/namespaces/{namespace}/tables/{table} + + - DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table} + + - POST /v1/{prefix}/namespaces/{namespace}/register + + - POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics + + - POST /v1/{prefix}/tables/rename + + - POST /v1/{prefix}/transactions/commit + " + responses: + 200: + description: Server specified configuration values. + content: + application/json: + schema: + $ref: '#/components/schemas/CatalogConfig' + example: { + "overrides": { + "warehouse": "s3://bucket/warehouse/" + }, + "defaults": { + "clients": "4" + }, + "endpoints": [ + "GET /v1/{prefix}/namespaces/{namespace}", + "GET /v1/{prefix}/namespaces", + "POST /v1/{prefix}/namespaces", + "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}", + "GET /v1/{prefix}/namespaces/{namespace}/views/{view}" + ] + } + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + /v1/oauth/tokens: + + post: + tags: + - OAuth2 API + summary: Get a token using an OAuth2 flow (DEPRECATED for REMOVAL) + deprecated: true + operationId: getToken + description: + The `oauth/tokens` endpoint is **DEPRECATED for REMOVAL**. It is _not_ recommended to + implement this endpoint, unless you are fully aware of the potential security implications. + + All clients are encouraged to explicitly set the configuration property `oauth2-server-uri` + to the correct OAuth endpoint. + + Deprecated since Iceberg (Java) 1.6.0. The endpoint and related types will be removed from + this spec in Iceberg (Java) 2.0. + + See [Security improvements in the Iceberg REST specification](https://github.com/apache/iceberg/issues/10537) + + + Exchange credentials for a token using the OAuth2 client credentials flow or token exchange. + + + This endpoint is used for three purposes - + + 1. To exchange client credentials (client ID and secret) for an access token + This uses the client credentials flow. + + 2. To exchange a client token and an identity token for a more specific access token + This uses the token exchange flow. + + 3. To exchange an access token for one with the same claims and a refreshed expiration period + This uses the token exchange flow. + + + For example, a catalog client may be configured with client credentials from the OAuth2 + Authorization flow. This client would exchange its client ID and secret for an access token + using the client credentials request with this endpoint (1). Subsequent requests would then + use that access token. + + + Some clients may also handle sessions that have additional user context. These clients would + use the token exchange flow to exchange a user token (the "subject" token) from the session + for a more specific access token for that user, using the catalog's access token as the + "actor" token (2). The user ID token is the "subject" token and can be any token type + allowed by the OAuth2 token exchange flow, including a unsecured JWT token with a sub claim. + This request should use the catalog's bearer token in the "Authorization" header. + + + Clients may also use the token exchange flow to refresh a token that is about to expire by + sending a token exchange request (3). The request's "subject" token should be the expiring + token. This request should use the subject token in the "Authorization" header. + requestBody: + required: true + content: + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/OAuthTokenRequest' + responses: + 200: + $ref: '#/components/responses/OAuthTokenResponse' + 400: + $ref: '#/components/responses/OAuthErrorResponse' + 401: + $ref: '#/components/responses/OAuthErrorResponse' + 5XX: + $ref: '#/components/responses/OAuthErrorResponse' + + /v1/{prefix}/namespaces: + parameters: + - $ref: '#/components/parameters/prefix' + + get: + tags: + - Catalog API + summary: List namespaces, optionally providing a parent namespace to list underneath + description: + List all namespaces at a certain level, optionally starting from a given parent namespace. + If table accounting.tax.paid.info exists, using 'SELECT NAMESPACE IN accounting' would + translate into `GET /namespaces?parent=accounting` and must return a namespace, ["accounting", "tax"] only. + Using 'SELECT NAMESPACE IN accounting.tax' would + translate into `GET /namespaces?parent=accounting%1Ftax` and must return a namespace, ["accounting", "tax", "paid"]. + If `parent` is not provided, all top-level namespaces should be listed. + operationId: listNamespaces + parameters: + - $ref: '#/components/parameters/page-token' + - $ref: '#/components/parameters/page-size' + - name: parent + in: query + description: + An optional namespace, underneath which to list namespaces. + If not provided or empty, all top-level namespaces should be listed. + If parent is a multipart namespace, the parts must be separated by the unit separator (`0x1F`) byte. + required: false + allowEmptyValue: true + schema: + type: string + example: "accounting%1Ftax" + responses: + 200: + $ref: '#/components/responses/ListNamespacesResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: Not Found - Namespace provided in the `parent` query parameter is not found. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NoSuchNamespaceExample: + $ref: '#/components/examples/NoSuchNamespaceError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + post: + tags: + - Catalog API + summary: Create a namespace + description: + Create a namespace, with an optional set of properties. + The server might also add properties, such as `last_modified_time` etc. + operationId: createNamespace + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CreateNamespaceRequest' + responses: + 200: + $ref: '#/components/responses/CreateNamespaceResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 406: + $ref: '#/components/responses/UnsupportedOperationResponse' + 409: + description: Conflict - The namespace already exists + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NamespaceAlreadyExists: + $ref: '#/components/examples/NamespaceAlreadyExistsError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + /v1/{prefix}/namespaces/{namespace}: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + + get: + tags: + - Catalog API + summary: Load the metadata properties for a namespace + operationId: loadNamespaceMetadata + description: Return all stored metadata properties for a given namespace + responses: + 200: + $ref: '#/components/responses/GetNamespaceResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: Not Found - Namespace not found + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NoSuchNamespaceExample: + $ref: '#/components/examples/NoSuchNamespaceError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + head: + tags: + - Catalog API + summary: Check if a namespace exists + operationId: namespaceExists + description: + Check if a namespace exists. The response does not contain a body. + responses: + 204: + description: Success, no content + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: Not Found - Namespace not found + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NoSuchNamespaceExample: + $ref: '#/components/examples/NoSuchNamespaceError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + delete: + tags: + - Catalog API + summary: Drop a namespace from the catalog. Namespace must be empty. + operationId: dropNamespace + responses: + 204: + description: Success, no content + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: Not Found - Namespace to delete does not exist. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NoSuchNamespaceExample: + $ref: '#/components/examples/NoSuchNamespaceError' + 409: + description: Not Empty - Namespace to delete is not empty. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NamespaceNotEmptyExample: + $ref: '#/components/examples/NamespaceNotEmptyError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + /v1/{prefix}/namespaces/{namespace}/properties: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + + post: + tags: + - Catalog API + summary: Set or remove properties on a namespace + operationId: updateProperties + description: + Set and/or remove properties on a namespace. + The request body specifies a list of properties to remove and a map + of key value pairs to update. + + Properties that are not in the request are not modified or removed by this call. + + Server implementations are not required to support namespace properties. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateNamespacePropertiesRequest' + examples: + UpdateAndRemoveProperties: + $ref: '#/components/examples/UpdateAndRemoveNamespacePropertiesRequest' + responses: + 200: + $ref: '#/components/responses/UpdateNamespacePropertiesResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: Not Found - Namespace not found + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NamespaceNotFound: + $ref: '#/components/examples/NoSuchNamespaceError' + 406: + $ref: '#/components/responses/UnsupportedOperationResponse' + 422: + description: Unprocessable Entity - A property key was included in both `removals` and `updates` + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + UnprocessableEntityDuplicateKey: + $ref: '#/components/examples/UnprocessableEntityDuplicateKey' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + /v1/{prefix}/namespaces/{namespace}/tables: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + + get: + tags: + - Catalog API + summary: List all table identifiers underneath a given namespace + description: Return all table identifiers under this namespace + operationId: listTables + parameters: + - $ref: '#/components/parameters/page-token' + - $ref: '#/components/parameters/page-size' + responses: + 200: + $ref: '#/components/responses/ListTablesResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NamespaceNotFound: + $ref: '#/components/examples/NoSuchNamespaceError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + post: + tags: + - Catalog API + summary: Create a table in the given namespace + description: + Create a table or start a create transaction, like atomic CTAS. + + + If `stage-create` is false, the table is created immediately. + + + If `stage-create` is true, the table is not created, but table metadata is initialized and returned. + The service should prepare as needed for a commit to the table commit endpoint to complete the create + transaction. The client uses the returned metadata to begin a transaction. To commit the transaction, + the client sends all create and subsequent changes to the table commit route. Changes from the table + create operation include changes like AddSchemaUpdate and SetCurrentSchemaUpdate that set the initial + table state. + operationId: createTable + parameters: + - $ref: '#/components/parameters/data-access' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CreateTableRequest' + responses: + 200: + $ref: '#/components/responses/CreateTableResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NamespaceNotFound: + $ref: '#/components/examples/NoSuchNamespaceError' + 409: + description: Conflict - The table already exists + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NamespaceAlreadyExists: + $ref: '#/components/examples/TableAlreadyExistsError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + /v1/{prefix}/namespaces/{namespace}/tables/{table}/plan: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + - $ref: '#/components/parameters/table' + post: + tags: + - Catalog API + summary: Submit a scan for planning + description: > + Submits a scan for server-side planning. + + + Point-in-time scans are planned by passing snapshot-id to identify the + table snapshot to scan. Incremental scans are planned by passing both + start-snapshot-id and end-snapshot-id. Requests that include both point + in time config properties and incremental config properties are + invalid. If the request does not include either incremental or + point-in-time config properties, scan planning should produce a + point-in-time scan of the latest snapshot in the table's main branch. + + + Responses must include a valid status listed below. A "cancelled" status is considered invalid for this endpoint. + + - When "completed" the planning operation has produced plan tasks and + file scan tasks that must be returned in the response (not fetched + later by calling fetchPlanningResult) + + - When "submitted" the response must include a plan-id used to poll + fetchPlanningResult to fetch the planning result when it is ready + + - When "failed" the response must be a valid error response + + The response for a "completed" planning operation includes two types of + tasks (file scan tasks and plan tasks) and both may be included in the + response. Tasks must not be included for any other response status. + + + Responses that include a plan-id indicate that the service is holding + state or performing work for the client. + + + - Clients should use the plan-id to fetch results from + fetchPlanningResult when the response status is "submitted" + + - Clients should inform the service if planning results are no longer + needed by calling cancelPlanning. Cancellation is not necessary after + fetchScanTasks has been used to fetch scan tasks for each plan task. + operationId: planTableScan + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PlanTableScanRequest' + responses: + 200: + $ref: '#/components/responses/PlanTableScanResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found + - NoSuchTableException, the table does not exist + - NoSuchNamespaceException, the namespace does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + NamespaceDoesNotExist: + $ref: '#/components/examples/NoSuchNamespaceError' + 406: + $ref: '#/components/responses/UnsupportedOperationResponse' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + /v1/{prefix}/namespaces/{namespace}/tables/{table}/plan/{plan-id}: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + - $ref: '#/components/parameters/table' + - $ref: '#/components/parameters/plan-id' + + get: + tags: + - Catalog API + summary: Fetches the result of scan planning for a plan-id + operationId: fetchPlanningResult + description: > + Fetches the result of scan planning for a plan-id. + + + Responses must include a valid status + + - When "completed" the planning operation has produced plan-tasks and + file-scan-tasks that must be returned in the response + + - When "submitted" the planning operation has not completed; the client + should wait to call this endpoint again to fetch a completed response + + - When "failed" the response must be a valid error response + + - When "cancelled" the plan-id is invalid and should be discarded + + + The response for a "completed" planning operation includes two types of + tasks (file scan tasks and plan tasks) and both may be included in the + response. Tasks must not be included for any other response status. + responses: + 200: + $ref: '#/components/responses/FetchPlanningResultResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found + - NoSuchPlanIdException, the plan-id does not exist + - NoSuchTableException, the table does not exist + - NoSuchNamespaceException, the namespace does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + PlanIdDoesNotExist: + $ref: '#/components/examples/NoSuchPlanIdError' + TableDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + NamespaceDoesNotExist: + $ref: '#/components/examples/NoSuchNamespaceError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + delete: + tags: + - Catalog API + summary: Cancels scan planning for a plan-id + operationId: cancelPlanning + description: > + Cancels scan planning for a plan-id. + + + This notifies the service that it can release resources held for the + scan. Clients should cancel scans that are no longer needed, either + while the plan-id returns a "submitted" status or while there are + remaining plan tasks that have not been fetched. + + + Cancellation is not necessary when + + - Scan tasks for each plan task have been fetched using fetchScanTasks + + - A plan-id has produced a "failed" or "cancelled" status from + planTableScan or fetchPlanningResult + responses: + 204: + description: Success, no content + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found + - NoSuchTableException, the table does not exist + - NoSuchNamespaceException, the namespace does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + NamespaceDoesNotExist: + $ref: '#/components/examples/NoSuchNamespaceError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + + /v1/{prefix}/namespaces/{namespace}/tables/{table}/tasks: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + - $ref: '#/components/parameters/table' + + post: + tags: + - Catalog API + summary: Fetches result tasks for a plan task + operationId: fetchScanTasks + description: Fetches result tasks for a plan task. + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/FetchScanTasksRequest' + responses: + 200: + $ref: '#/components/responses/FetchScanTasksResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found + - NoSuchPlanTaskException, the plan-task does not exist + - NoSuchTableException, the table does not exist + - NoSuchNamespaceException, the namespace does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + PlanTaskDoesNotExist: + $ref: '#/components/examples/NoSuchPlanTaskError' + TableDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + NamespaceDoesNotExist: + $ref: '#/components/examples/NoSuchNamespaceError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + + + /v1/{prefix}/namespaces/{namespace}/register: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + + post: + tags: + - Catalog API + summary: Register a table in the given namespace using given metadata file location + description: + Register a table using given metadata file location. + + operationId: registerTable + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/RegisterTableRequest' + responses: + 200: + $ref: '#/components/responses/LoadTableResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NamespaceNotFound: + $ref: '#/components/examples/NoSuchNamespaceError' + 409: + description: Conflict - The table already exists + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NamespaceAlreadyExists: + $ref: '#/components/examples/TableAlreadyExistsError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + /v1/{prefix}/namespaces/{namespace}/tables/{table}: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + - $ref: '#/components/parameters/table' + + get: + tags: + - Catalog API + summary: Load a table from the catalog + operationId: loadTable + description: + Load a table from the catalog. + + + The response contains both configuration and table metadata. The configuration, if non-empty is used + as additional configuration for the table that overrides catalog configuration. For example, this + configuration may change the FileIO implementation to be used for the table. + + + The response also contains the table's full metadata, matching the table metadata JSON file. + + + The catalog configuration may contain credentials that should be used for subsequent requests for the + table. The configuration key "token" is used to pass an access token to be used as a bearer token + for table requests. Otherwise, a token may be passed using a RFC 8693 token type as a configuration + key. For example, "urn:ietf:params:oauth:token-type:jwt=". + parameters: + - $ref: '#/components/parameters/data-access' + - name: If-None-Match + in: header + description: + An optional header that allows the server to return 304 (Not Modified) if the metadata + is current. The content is the value of the ETag received in a CreateTableResponse or + LoadTableResponse. + required: false + schema: + type: string + - in: query + name: snapshots + description: + The snapshots to return in the body of the metadata. Setting the value to `all` would + return the full set of snapshots currently valid for the table. Setting the value to + `refs` would load all snapshots referenced by branches or tags. + + Default if no param is provided is `all`. + required: false + schema: + type: string + enum: [ all, refs ] + responses: + 200: + $ref: '#/components/responses/LoadTableResponse' + 304: + description: + Not Modified - Based on the content of the 'If-None-Match' header the table metadata has + not changed since. + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableException, table to load does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToLoadDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + post: + tags: + - Catalog API + summary: Commit updates to a table + operationId: updateTable + description: + Commit updates to a table. + + + Commits have two parts, requirements and updates. Requirements are assertions that will be validated + before attempting to make and commit changes. For example, `assert-ref-snapshot-id` will check that a + named ref's snapshot ID has a certain value. + Server implementations are required to fail with a 400 status code + if any unknown updates or requirements are received. + + + Updates are changes to make to table metadata. For example, after asserting that the current main ref + is at the expected snapshot, a commit may add a new child snapshot and set the ref to the new + snapshot id. + + + Create table transactions that are started by createTable with `stage-create` set to true are + committed using this route. Transactions should include all changes to the table, including table + initialization, like AddSchemaUpdate and SetCurrentSchemaUpdate. The `assert-create` requirement is + used to ensure that the table was not created concurrently. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CommitTableRequest' + responses: + 200: + $ref: '#/components/responses/CommitTableResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableException, table to load does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToUpdateDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + 409: + description: + Conflict - CommitFailedException, one or more requirements failed. The client may retry. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 500: + description: + An unknown server-side problem occurred; the commit state is unknown. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Internal Server Error", + "type": "CommitStateUnknownException", + "code": 500 + } + } + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 502: + description: + A gateway or proxy received an invalid response from the upstream server; the commit state is unknown. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Invalid response from the upstream server", + "type": "CommitStateUnknownException", + "code": 502 + } + } + 504: + description: + A server-side gateway timeout occurred; the commit state is unknown. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Gateway timed out during commit", + "type": "CommitStateUnknownException", + "code": 504 + } + } + 5XX: + description: + A server-side problem that might not be addressable on the client. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Bad Gateway", + "type": "InternalServerError", + "code": 502 + } + } + + delete: + tags: + - Catalog API + summary: Drop a table from the catalog + operationId: dropTable + description: Remove a table from the catalog + parameters: + - name: purgeRequested + in: query + required: false + description: Whether the user requested to purge the underlying table's data and metadata + schema: + type: boolean + default: false + responses: + 204: + description: Success, no content + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableException, Table to drop does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToDeleteDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + head: + tags: + - Catalog API + summary: Check if a table exists + operationId: tableExists + description: + Check if a table exists within a given namespace. The response does not contain a body. + responses: + 204: + description: Success, no content + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableException, Table not found + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToLoadDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + /v1/{prefix}/namespaces/{namespace}/tables/{table}/credentials: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + - $ref: '#/components/parameters/table' + + get: + tags: + - Catalog API + summary: Load vended credentials for a table from the catalog + operationId: loadCredentials + description: Load vended credentials for a table from the catalog. + responses: + 200: + $ref: '#/components/responses/LoadCredentialsResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableException, table to load credentials for does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToLoadDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + /v1/{prefix}/tables/rename: + parameters: + - $ref: '#/components/parameters/prefix' + + post: + tags: + - Catalog API + summary: Rename a table from its current name to a new name + description: + Rename a table from one identifier to another. It's valid to move a table + across namespaces, but the server implementation is not required to support it. + operationId: renameTable + requestBody: + description: Current table identifier to rename and new table identifier to rename to + content: + application/json: + schema: + $ref: '#/components/schemas/RenameTableRequest' + examples: + RenameTableSameNamespace: + $ref: '#/components/examples/RenameTableSameNamespace' + required: true + responses: + 204: + description: Success, no content + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found + - NoSuchTableException, Table to rename does not exist + - NoSuchNamespaceException, The target namespace of the new table identifier does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToRenameDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + NamespaceToRenameToDoesNotExist: + $ref: '#/components/examples/NoSuchNamespaceError' + 406: + $ref: '#/components/responses/UnsupportedOperationResponse' + 409: + description: Conflict - The target identifier to rename to already exists as a table or view + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: + $ref: '#/components/examples/TableAlreadyExistsError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + - $ref: '#/components/parameters/table' + + post: + tags: + - Catalog API + summary: Send a metrics report to this endpoint to be processed by the backend + operationId: reportMetrics + requestBody: + description: The request containing the metrics report to be sent + content: + application/json: + schema: + $ref: '#/components/schemas/ReportMetricsRequest' + required: true + responses: + 204: + description: Success, no content + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableException, table to load does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToLoadDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + /v1/{prefix}/transactions/commit: + parameters: + - $ref: '#/components/parameters/prefix' + + post: + tags: + - Catalog API + summary: Commit updates to multiple tables in an atomic operation + operationId: commitTransaction + requestBody: + description: + Commit updates to multiple tables in an atomic operation + + + A commit for a single table consists of a table identifier with requirements and updates. + Requirements are assertions that will be validated before attempting to make and commit changes. + For example, `assert-ref-snapshot-id` will check that a named ref's snapshot ID has a certain value. + Server implementations are required to fail with a 400 status code + if any unknown updates or requirements are received. + + Updates are changes to make to table metadata. For example, after asserting that the current main ref + is at the expected snapshot, a commit may add a new child snapshot and set the ref to the new + snapshot id. + content: + application/json: + schema: + $ref: '#/components/schemas/CommitTransactionRequest' + required: true + responses: + 204: + description: Success, no content + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableException, table to load does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToUpdateDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + 409: + description: + Conflict - CommitFailedException, one or more requirements failed. The client may retry. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 500: + description: + An unknown server-side problem occurred; the commit state is unknown. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Internal Server Error", + "type": "CommitStateUnknownException", + "code": 500 + } + } + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 502: + description: + A gateway or proxy received an invalid response from the upstream server; the commit state is unknown. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Invalid response from the upstream server", + "type": "CommitStateUnknownException", + "code": 502 + } + } + 504: + description: + A server-side gateway timeout occurred; the commit state is unknown. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Gateway timed out during commit", + "type": "CommitStateUnknownException", + "code": 504 + } + } + 5XX: + description: + A server-side problem that might not be addressable on the client. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Bad Gateway", + "type": "InternalServerError", + "code": 502 + } + } + + /v1/{prefix}/namespaces/{namespace}/views: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + + get: + tags: + - Catalog API + summary: List all view identifiers underneath a given namespace + description: Return all view identifiers under this namespace + operationId: listViews + parameters: + - $ref: '#/components/parameters/page-token' + - $ref: '#/components/parameters/page-size' + responses: + 200: + $ref: '#/components/responses/ListTablesResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + examples: + NamespaceNotFound: + $ref: '#/components/examples/NoSuchNamespaceError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + post: + tags: + - Catalog API + summary: Create a view in the given namespace + description: + Create a view in the given namespace. + operationId: createView + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CreateViewRequest' + responses: + 200: + $ref: '#/components/responses/LoadViewResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + examples: + NamespaceNotFound: + $ref: '#/components/examples/NoSuchNamespaceError' + 409: + description: Conflict - The view already exists + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + examples: + NamespaceAlreadyExists: + $ref: '#/components/examples/ViewAlreadyExistsError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + /v1/{prefix}/namespaces/{namespace}/views/{view}: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + - $ref: '#/components/parameters/view' + + get: + tags: + - Catalog API + summary: Load a view from the catalog + operationId: loadView + description: + Load a view from the catalog. + + + The response contains both configuration and view metadata. The configuration, if non-empty is used + as additional configuration for the view that overrides catalog configuration. + + + The response also contains the view's full metadata, matching the view metadata JSON file. + + + The catalog configuration may contain credentials that should be used for subsequent requests for the + view. The configuration key "token" is used to pass an access token to be used as a bearer token + for view requests. Otherwise, a token may be passed using a RFC 8693 token type as a configuration + key. For example, "urn:ietf:params:oauth:token-type:jwt=". + responses: + 200: + $ref: '#/components/responses/LoadViewResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchViewException, view to load does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + examples: + ViewToLoadDoesNotExist: + $ref: '#/components/examples/NoSuchViewError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + post: + tags: + - Catalog API + summary: Replace a view + operationId: replaceView + description: + Commit updates to a view. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CommitViewRequest' + responses: + 200: + $ref: '#/components/responses/LoadViewResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchViewException, view to load does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + examples: + ViewToUpdateDoesNotExist: + $ref: '#/components/examples/NoSuchViewError' + 409: + description: + Conflict - CommitFailedException. The client may retry. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 500: + description: + An unknown server-side problem occurred; the commit state is unknown. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + example: { + "error": { + "message": "Internal Server Error", + "type": "CommitStateUnknownException", + "code": 500 + } + } + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 502: + description: + A gateway or proxy received an invalid response from the upstream server; the commit state is unknown. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + example: { + "error": { + "message": "Invalid response from the upstream server", + "type": "CommitStateUnknownException", + "code": 502 + } + } + 504: + description: + A server-side gateway timeout occurred; the commit state is unknown. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + example: { + "error": { + "message": "Gateway timed out during commit", + "type": "CommitStateUnknownException", + "code": 504 + } + } + 5XX: + description: + A server-side problem that might not be addressable on the client. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + example: { + "error": { + "message": "Bad Gateway", + "type": "InternalServerError", + "code": 502 + } + } + + delete: + tags: + - Catalog API + summary: Drop a view from the catalog + operationId: dropView + description: Remove a view from the catalog + responses: + 204: + description: Success, no content + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchViewException, view to drop does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + examples: + ViewToDeleteDoesNotExist: + $ref: '#/components/examples/NoSuchViewError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + head: + tags: + - Catalog API + summary: Check if a view exists + operationId: viewExists + description: + Check if a view exists within a given namespace. This request does not return a response body. + responses: + 204: + description: Success, no content + 400: + description: Bad Request + 401: + description: Unauthorized + 404: + description: Not Found + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + /v1/{prefix}/views/rename: + parameters: + - $ref: '#/components/parameters/prefix' + + post: + tags: + - Catalog API + summary: Rename a view from its current name to a new name + description: + Rename a view from one identifier to another. It's valid to move a view + across namespaces, but the server implementation is not required to support it. + operationId: renameView + requestBody: + description: Current view identifier to rename and new view identifier to rename to + content: + application/json: + schema: + $ref: '#/components/schemas/RenameTableRequest' + examples: + RenameViewSameNamespace: + $ref: '#/components/examples/RenameViewSameNamespace' + required: true + responses: + 204: + description: Success, no content + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found + - NoSuchViewException, view to rename does not exist + - NoSuchNamespaceException, The target namespace of the new identifier does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + examples: + ViewToRenameDoesNotExist: + $ref: '#/components/examples/NoSuchViewError' + NamespaceToRenameToDoesNotExist: + $ref: '#/components/examples/NoSuchNamespaceError' + 406: + $ref: '#/components/responses/UnsupportedOperationResponse' + 409: + description: Conflict - The target identifier to rename to already exists as a table or view + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + example: + $ref: '#/components/examples/ViewAlreadyExistsError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + +components: + ####################################################### + # Common Parameter Definitions Used In Several Routes # + ####################################################### + parameters: + namespace: + name: namespace + in: path + required: true + description: + A namespace identifier as a single string. + Multipart namespace parts should be separated by the unit separator (`0x1F`) byte. + schema: + type: string + examples: + singlepart_namespace: + value: "accounting" + multipart_namespace: + value: "accounting%1Ftax" + + prefix: + name: prefix + in: path + schema: + type: string + required: true + description: An optional prefix in the path + + table: + name: table + in: path + description: A table name + required: true + schema: + type: string + example: "sales" + + plan-id: + name: plan-id + in: path + description: ID used to track a planning request + required: true + schema: + type: string + + view: + name: view + in: path + description: A view name + required: true + schema: + type: string + example: "sales" + + data-access: + name: X-Iceberg-Access-Delegation + in: header + description: > + Optional signal to the server that the client supports delegated access + via a comma-separated list of access mechanisms. The server may choose + to supply access via any or none of the requested mechanisms. + + + Specific properties and handling for `vended-credentials` is documented + in the `LoadTableResult` schema section of this spec document. + + + The protocol and specification for `remote-signing` is documented in + the `s3-signer-open-api.yaml` OpenApi spec in the `aws` module. + + required: false + schema: + type: string + enum: + - vended-credentials + - remote-signing + style: simple + explode: false + example: "vended-credentials,remote-signing" + + page-token: + name: pageToken + in: query + required: false + allowEmptyValue: true + schema: + $ref: '#/components/schemas/PageToken' + + page-size: + name: pageSize + in: query + description: + For servers that support pagination, this signals an upper bound of the number of results that a client will receive. + For servers that do not support pagination, clients may receive results larger than the indicated `pageSize`. + required: false + schema: + type: integer + minimum: 1 + + etag: + name: ETag + in: header + description: Identifies a unique version of the table metadata. + required: false + schema: + type: string + + ############################## + # Application Schema Objects # + ############################## + schemas: + + ErrorModel: + type: object + description: JSON error payload returned in a response with further details on the error + required: + - message + - type + - code + properties: + message: + type: string + description: Human-readable error message + type: + type: string + description: Internal type definition of the error + example: NoSuchNamespaceException + code: + type: integer + minimum: 400 + maximum: 600 + description: HTTP response code + example: 404 + stack: + type: array + items: + type: string + + CatalogConfig: + type: object + description: Server-provided configuration for the catalog. + required: + - defaults + - overrides + properties: + overrides: + type: object + additionalProperties: + type: string + description: + Properties that should be used to override client configuration; applied after defaults and client configuration. + defaults: + type: object + additionalProperties: + type: string + description: + Properties that should be used as default configuration; applied before client configuration. + endpoints: + type: array + items: + type: string + description: A list of endpoints that the server supports. The format of each endpoint must be " ". + The HTTP verb and the resource path must be separated by a space character. + example: [ + "GET /v1/{prefix}/namespaces/{namespace}", + "GET /v1/{prefix}/namespaces", + "POST /v1/{prefix}/namespaces", + "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}", + "GET /v1/{prefix}/namespaces/{namespace}/views/{view}" + ] + + CreateNamespaceRequest: + type: object + required: + - namespace + properties: + namespace: + $ref: '#/components/schemas/Namespace' + properties: + type: object + description: Configured string to string map of properties for the namespace + example: { "owner": "Hank Bendickson" } + default: { } + additionalProperties: + type: string + + UpdateNamespacePropertiesRequest: + type: object + properties: + removals: + type: array + uniqueItems: true + items: + type: string + example: [ "department", "access_group" ] + updates: + type: object + example: { "owner": "Hank Bendickson" } + additionalProperties: + type: string + + RenameTableRequest: + type: object + required: + - source + - destination + properties: + source: + $ref: '#/components/schemas/TableIdentifier' + destination: + $ref: '#/components/schemas/TableIdentifier' + + Namespace: + description: Reference to one or more levels of a namespace + type: array + items: + type: string + example: [ "accounting", "tax" ] + + PageToken: + description: + An opaque token that allows clients to make use of pagination for list APIs + (e.g. ListTables). Clients may initiate the first paginated request by sending an empty + query parameter `pageToken` to the server. + + Servers that support pagination should identify the `pageToken` parameter and return a + `next-page-token` in the response if there are more results available. After the initial + request, the value of `next-page-token` from each response must be used as the `pageToken` + parameter value for the next request. The server must return `null` value for the + `next-page-token` in the last response. + + Servers that support pagination must return all results in a single response with the value + of `next-page-token` set to `null` if the query parameter `pageToken` is not set in the + request. + + Servers that do not support pagination should ignore the `pageToken` parameter and return + all results in a single response. The `next-page-token` must be omitted from the response. + + Clients must interpret either `null` or missing response value of `next-page-token` as + the end of the listing results. + + type: string + nullable: true + + TableIdentifier: + type: object + required: + - namespace + - name + properties: + namespace: + $ref: '#/components/schemas/Namespace' + name: + type: string + nullable: false + + PrimitiveType: + type: string + example: + - "long" + - "string" + - "fixed[16]" + - "decimal(10,2)" + + StructField: + type: object + required: + - id + - name + - type + - required + properties: + id: + type: integer + name: + type: string + type: + $ref: '#/components/schemas/Type' + required: + type: boolean + doc: + type: string + initial-default: + $ref: "#/components/schemas/PrimitiveTypeValue" + write-default: + $ref: "#/components/schemas/PrimitiveTypeValue" + + StructType: + type: object + required: + - type + - fields + properties: + type: + type: string + const: "struct" + fields: + type: array + items: + $ref: '#/components/schemas/StructField' + + ListType: + type: object + required: + - type + - element-id + - element + - element-required + properties: + type: + type: string + const: "list" + element-id: + type: integer + element: + $ref: '#/components/schemas/Type' + element-required: + type: boolean + + MapType: + type: object + required: + - type + - key-id + - key + - value-id + - value + - value-required + properties: + type: + type: string + const: "map" + key-id: + type: integer + key: + $ref: '#/components/schemas/Type' + value-id: + type: integer + value: + $ref: '#/components/schemas/Type' + value-required: + type: boolean + + Type: + oneOf: + - $ref: '#/components/schemas/PrimitiveType' + - $ref: '#/components/schemas/StructType' + - $ref: '#/components/schemas/ListType' + - $ref: '#/components/schemas/MapType' + + Schema: + allOf: + - $ref: '#/components/schemas/StructType' + - type: object + properties: + schema-id: + type: integer + readOnly: true + identifier-field-ids: + type: array + items: + type: integer + + Expression: + oneOf: + - $ref: '#/components/schemas/TrueExpression' + - $ref: '#/components/schemas/FalseExpression' + - $ref: '#/components/schemas/AndOrExpression' + - $ref: '#/components/schemas/NotExpression' + - $ref: '#/components/schemas/SetExpression' + - $ref: '#/components/schemas/LiteralExpression' + - $ref: '#/components/schemas/UnaryExpression' + + ExpressionType: + type: string + example: + - "true" + - "false" + - "eq" + - "and" + - "or" + - "not" + - "in" + - "not-in" + - "lt" + - "lt-eq" + - "gt" + - "gt-eq" + - "not-eq" + - "starts-with" + - "not-starts-with" + - "is-null" + - "not-null" + - "is-nan" + - "not-nan" + + TrueExpression: + type: object + required: + - type + properties: + type: + $ref: '#/components/schemas/ExpressionType' + const: "true" + + FalseExpression: + type: object + required: + - type + properties: + type: + $ref: '#/components/schemas/ExpressionType' + const: "false" + + AndOrExpression: + type: object + required: + - type + - left + - right + properties: + type: + $ref: '#/components/schemas/ExpressionType' + enum: ["and", "or"] + left: + $ref: '#/components/schemas/Expression' + right: + $ref: '#/components/schemas/Expression' + + NotExpression: + type: object + required: + - type + - child + properties: + type: + $ref: '#/components/schemas/ExpressionType' + const: "not" + child: + $ref: '#/components/schemas/Expression' + + UnaryExpression: + type: object + required: + - type + - term + - value + properties: + type: + $ref: '#/components/schemas/ExpressionType' + enum: ["is-null", "not-null", "is-nan", "not-nan"] + term: + $ref: '#/components/schemas/Term' + value: + type: object + + LiteralExpression: + type: object + required: + - type + - term + - value + properties: + type: + $ref: '#/components/schemas/ExpressionType' + enum: ["lt", "lt-eq", "gt", "gt-eq", "eq", "not-eq", "starts-with", "not-starts-with"] + term: + $ref: '#/components/schemas/Term' + value: + type: object + + SetExpression: + type: object + required: + - type + - term + - values + properties: + type: + $ref: '#/components/schemas/ExpressionType' + enum: ["in", "not-in"] + term: + $ref: '#/components/schemas/Term' + values: + type: array + items: + type: object + + Term: + oneOf: + - $ref: '#/components/schemas/Reference' + - $ref: '#/components/schemas/TransformTerm' + + Reference: + type: string + example: + - "column-name" + + TransformTerm: + type: object + required: + - type + - transform + - term + properties: + type: + type: string + const: "transform" + transform: + $ref: '#/components/schemas/Transform' + term: + $ref: '#/components/schemas/Reference' + + Transform: + type: string + example: + - "identity" + - "year" + - "month" + - "day" + - "hour" + - "bucket[256]" + - "truncate[16]" + + PartitionField: + type: object + required: + - source-id + - transform + - name + properties: + field-id: + type: integer + source-id: + type: integer + name: + type: string + transform: + $ref: '#/components/schemas/Transform' + + PartitionSpec: + type: object + required: + - fields + properties: + spec-id: + type: integer + readOnly: true + fields: + type: array + items: + $ref: '#/components/schemas/PartitionField' + + SortDirection: + type: string + enum: ["asc", "desc"] + + NullOrder: + type: string + enum: ["nulls-first", "nulls-last"] + + SortField: + type: object + required: + - source-id + - transform + - direction + - null-order + properties: + source-id: + type: integer + transform: + $ref: '#/components/schemas/Transform' + direction: + $ref: '#/components/schemas/SortDirection' + null-order: + $ref: '#/components/schemas/NullOrder' + + SortOrder: + type: object + required: + - order-id + - fields + properties: + order-id: + type: integer + readOnly: true + fields: + type: array + items: + $ref: '#/components/schemas/SortField' + + EncryptedKey: + type: object + required: + - key-id + - encrypted-key-metadata + properties: + key-id: + type: string + encrypted-key-metadata: + type: string + format: byte # for compatibility + contentEncoding: base64 + encrypted-by-id: + type: string + properties: + type: object + additionalProperties: + type: string + + Snapshot: + type: object + required: + - snapshot-id + - timestamp-ms + - manifest-list + - summary + properties: + snapshot-id: + type: integer + format: int64 + parent-snapshot-id: + type: integer + format: int64 + sequence-number: + type: integer + format: int64 + timestamp-ms: + type: integer + format: int64 + manifest-list: + type: string + description: Location of the snapshot's manifest list file + first-row-id: + type: integer + format: int64 + description: The first _row_id assigned to the first row in the first data file in the first manifest + summary: + type: object + required: + - operation + properties: + operation: + type: string + enum: ["append", "replace", "overwrite", "delete"] + additionalProperties: + type: string + schema-id: + type: integer + + SnapshotReference: + type: object + required: + - type + - snapshot-id + properties: + type: + type: string + enum: ["tag", "branch"] + snapshot-id: + type: integer + format: int64 + max-ref-age-ms: + type: integer + format: int64 + max-snapshot-age-ms: + type: integer + format: int64 + min-snapshots-to-keep: + type: integer + + SnapshotReferences: + type: object + additionalProperties: + $ref: '#/components/schemas/SnapshotReference' + + SnapshotLog: + type: array + items: + type: object + required: + - snapshot-id + - timestamp-ms + properties: + snapshot-id: + type: integer + format: int64 + timestamp-ms: + type: integer + format: int64 + + MetadataLog: + type: array + items: + type: object + required: + - metadata-file + - timestamp-ms + properties: + metadata-file: + type: string + timestamp-ms: + type: integer + format: int64 + + TableMetadata: + type: object + required: + - format-version + - table-uuid + properties: + format-version: + type: integer + minimum: 1 + maximum: 3 + table-uuid: + type: string + location: + type: string + last-updated-ms: + type: integer + format: int64 + next-row-id: + type: integer + format: int64 + description: A long higher than all assigned row IDs; the next snapshot's first-row-id. + properties: + type: object + additionalProperties: + type: string + # schema tracking + schemas: + type: array + items: + $ref: '#/components/schemas/Schema' + current-schema-id: + type: integer + last-column-id: + type: integer + # partition spec tracking + partition-specs: + type: array + items: + $ref: '#/components/schemas/PartitionSpec' + default-spec-id: + type: integer + last-partition-id: + type: integer + # sort order tracking + sort-orders: + type: array + items: + $ref: '#/components/schemas/SortOrder' + default-sort-order-id: + type: integer + # encryption + encryption-keys: + type: array + items: + $ref: '#/components/schemas/EncryptedKey' + # snapshot tracking + snapshots: + type: array + items: + $ref: '#/components/schemas/Snapshot' + refs: + $ref: '#/components/schemas/SnapshotReferences' + current-snapshot-id: + type: integer + format: int64 + last-sequence-number: + type: integer + format: int64 + # logs + snapshot-log: + $ref: '#/components/schemas/SnapshotLog' + metadata-log: + $ref: '#/components/schemas/MetadataLog' + # statistics + statistics: + type: array + items: + $ref: '#/components/schemas/StatisticsFile' + partition-statistics: + type: array + items: + $ref: '#/components/schemas/PartitionStatisticsFile' + + SQLViewRepresentation: + type: object + required: + - type + - sql + - dialect + properties: + type: + type: string + sql: + type: string + dialect: + type: string + + ViewRepresentation: + oneOf: + - $ref: '#/components/schemas/SQLViewRepresentation' + + ViewHistoryEntry: + type: object + required: + - version-id + - timestamp-ms + properties: + version-id: + type: integer + timestamp-ms: + type: integer + format: int64 + + ViewVersion: + type: object + required: + - version-id + - timestamp-ms + - schema-id + - summary + - representations + - default-namespace + properties: + version-id: + type: integer + timestamp-ms: + type: integer + format: int64 + schema-id: + type: integer + description: Schema ID to set as current, or -1 to set last added schema + summary: + type: object + additionalProperties: + type: string + representations: + type: array + items: + $ref: '#/components/schemas/ViewRepresentation' + default-catalog: + type: string + default-namespace: + $ref: '#/components/schemas/Namespace' + + ViewMetadata: + type: object + required: + - view-uuid + - format-version + - location + - current-version-id + - versions + - version-log + - schemas + properties: + view-uuid: + type: string + format-version: + type: integer + minimum: 1 + maximum: 1 + location: + type: string + current-version-id: + type: integer + versions: + type: array + items: + $ref: '#/components/schemas/ViewVersion' + version-log: + type: array + items: + $ref: '#/components/schemas/ViewHistoryEntry' + schemas: + type: array + items: + $ref: '#/components/schemas/Schema' + properties: + type: object + additionalProperties: + type: string + + BaseUpdate: + discriminator: + propertyName: action + mapping: + assign-uuid: '#/components/schemas/AssignUUIDUpdate' + upgrade-format-version: '#/components/schemas/UpgradeFormatVersionUpdate' + add-schema: '#/components/schemas/AddSchemaUpdate' + set-current-schema: '#/components/schemas/SetCurrentSchemaUpdate' + add-spec: '#/components/schemas/AddPartitionSpecUpdate' + set-default-spec: '#/components/schemas/SetDefaultSpecUpdate' + add-sort-order: '#/components/schemas/AddSortOrderUpdate' + set-default-sort-order: '#/components/schemas/SetDefaultSortOrderUpdate' + add-snapshot: '#/components/schemas/AddSnapshotUpdate' + set-snapshot-ref: '#/components/schemas/SetSnapshotRefUpdate' + remove-snapshots: '#/components/schemas/RemoveSnapshotsUpdate' + remove-snapshot-ref: '#/components/schemas/RemoveSnapshotRefUpdate' + set-location: '#/components/schemas/SetLocationUpdate' + set-properties: '#/components/schemas/SetPropertiesUpdate' + remove-properties: '#/components/schemas/RemovePropertiesUpdate' + add-view-version: '#/components/schemas/AddViewVersionUpdate' + set-current-view-version: '#/components/schemas/SetCurrentViewVersionUpdate' + set-statistics: '#/components/schemas/SetStatisticsUpdate' + remove-statistics: '#/components/schemas/RemoveStatisticsUpdate' + set-partition-statistics: '#/components/schemas/SetPartitionStatisticsUpdate' + remove-partition-statistics: '#/components/schemas/RemovePartitionStatisticsUpdate' + remove-partition-specs: '#/components/schemas/RemovePartitionSpecsUpdate' + remove-schemas: '#/components/schemas/RemoveSchemasUpdate' + add-encryption-key: '#/components/schemas/AddEncryptionKeyUpdate' + remove-encryption-key: '#/components/schemas/RemoveEncryptionKeyUpdate' + type: object + required: + - action + properties: + action: + type: string + + AssignUUIDUpdate: + description: Assigning a UUID to a table/view should only be done when creating the table/view. It is not safe to re-assign the UUID if a table/view already has a UUID assigned + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - uuid + properties: + action: + type: string + const: "assign-uuid" + uuid: + type: string + + UpgradeFormatVersionUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - format-version + properties: + action: + type: string + const: "upgrade-format-version" + format-version: + type: integer + + AddSchemaUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - schema + properties: + action: + type: string + const: "add-schema" + schema: + $ref: '#/components/schemas/Schema' + last-column-id: + type: integer + deprecated: true + description: + This optional field is **DEPRECATED for REMOVAL** since it more safe to handle this internally, + and shouldn't be exposed to the clients. + + The highest assigned column ID for the table. This is used to ensure columns are always + assigned an unused ID when evolving schemas. When omitted, it will be computed on the server side. + + SetCurrentSchemaUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - schema-id + properties: + action: + type: string + const: "set-current-schema" + schema-id: + type: integer + description: Schema ID to set as current, or -1 to set last added schema + + AddPartitionSpecUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - spec + properties: + action: + type: string + const: "add-spec" + spec: + $ref: '#/components/schemas/PartitionSpec' + + SetDefaultSpecUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - spec-id + properties: + action: + type: string + const: "set-default-spec" + spec-id: + type: integer + description: Partition spec ID to set as the default, or -1 to set last added spec + + AddSortOrderUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - sort-order + properties: + action: + type: string + const: "add-sort-order" + sort-order: + $ref: '#/components/schemas/SortOrder' + + SetDefaultSortOrderUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - sort-order-id + properties: + action: + type: string + const: "set-default-sort-order" + sort-order-id: + type: integer + description: Sort order ID to set as the default, or -1 to set last added sort order + + AddSnapshotUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - snapshot + properties: + action: + type: string + const: "add-snapshot" + snapshot: + $ref: '#/components/schemas/Snapshot' + + SetSnapshotRefUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + - $ref: '#/components/schemas/SnapshotReference' + required: + - ref-name + properties: + action: + type: string + const: "set-snapshot-ref" + ref-name: + type: string + + RemoveSnapshotsUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - snapshot-ids + properties: + action: + type: string + const: "remove-snapshots" + snapshot-ids: + type: array + items: + type: integer + format: int64 + + RemoveSnapshotRefUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - ref-name + properties: + action: + type: string + const: "remove-snapshot-ref" + ref-name: + type: string + + SetLocationUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - location + properties: + action: + type: string + const: "set-location" + location: + type: string + + SetPropertiesUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - updates + properties: + action: + type: string + const: "set-properties" + updates: + type: object + additionalProperties: + type: string + + RemovePropertiesUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - removals + properties: + action: + type: string + const: "remove-properties" + removals: + type: array + items: + type: string + + AddViewVersionUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - view-version + properties: + action: + type: string + const: "add-view-version" + view-version: + $ref: '#/components/schemas/ViewVersion' + + SetCurrentViewVersionUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - view-version-id + properties: + action: + type: string + const: "set-current-view-version" + view-version-id: + type: integer + description: The view version id to set as current, or -1 to set last added view version id + + SetStatisticsUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - statistics + properties: + action: + type: string + const: "set-statistics" + snapshot-id: + type: integer + format: int64 + deprecated: true + description: + This optional field is **DEPRECATED for REMOVAL** since it contains redundant information. + Clients should use the `statistics.snapshot-id` field instead. + statistics: + $ref: '#/components/schemas/StatisticsFile' + + RemoveStatisticsUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - snapshot-id + properties: + action: + type: string + const: "remove-statistics" + snapshot-id: + type: integer + format: int64 + + SetPartitionStatisticsUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - partition-statistics + properties: + action: + type: string + const: "set-partition-statistics" + partition-statistics: + $ref: '#/components/schemas/PartitionStatisticsFile' + + RemovePartitionStatisticsUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - snapshot-id + properties: + action: + type: string + const: "remove-partition-statistics" + snapshot-id: + type: integer + format: int64 + + RemovePartitionSpecsUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - spec-ids + properties: + action: + type: string + const: "remove-partition-specs" + spec-ids: + type: array + items: + type: integer + + RemoveSchemasUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - schema-ids + properties: + action: + type: string + const: "remove-schemas" + schema-ids: + type: array + items: + type: integer + + AddEncryptionKeyUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - encryption-key + properties: + action: + type: string + const: "add-encryption-key" + encryption-key: + $ref: '#/components/schemas/EncryptedKey' + + RemoveEncryptionKeyUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + required: + - key-id + properties: + action: + type: string + const: "remove-encryption-key" + key-id: + type: string + + TableUpdate: + anyOf: + - $ref: '#/components/schemas/AssignUUIDUpdate' + - $ref: '#/components/schemas/UpgradeFormatVersionUpdate' + - $ref: '#/components/schemas/AddSchemaUpdate' + - $ref: '#/components/schemas/SetCurrentSchemaUpdate' + - $ref: '#/components/schemas/AddPartitionSpecUpdate' + - $ref: '#/components/schemas/SetDefaultSpecUpdate' + - $ref: '#/components/schemas/AddSortOrderUpdate' + - $ref: '#/components/schemas/SetDefaultSortOrderUpdate' + - $ref: '#/components/schemas/AddSnapshotUpdate' + - $ref: '#/components/schemas/SetSnapshotRefUpdate' + - $ref: '#/components/schemas/RemoveSnapshotsUpdate' + - $ref: '#/components/schemas/RemoveSnapshotRefUpdate' + - $ref: '#/components/schemas/SetLocationUpdate' + - $ref: '#/components/schemas/SetPropertiesUpdate' + - $ref: '#/components/schemas/RemovePropertiesUpdate' + - $ref: '#/components/schemas/SetStatisticsUpdate' + - $ref: '#/components/schemas/RemoveStatisticsUpdate' + - $ref: '#/components/schemas/RemovePartitionSpecsUpdate' + - $ref: '#/components/schemas/RemoveSchemasUpdate' + - $ref: '#/components/schemas/AddEncryptionKeyUpdate' + - $ref: '#/components/schemas/RemoveEncryptionKeyUpdate' + + ViewUpdate: + anyOf: + - $ref: '#/components/schemas/AssignUUIDUpdate' + - $ref: '#/components/schemas/UpgradeFormatVersionUpdate' + - $ref: '#/components/schemas/AddSchemaUpdate' + - $ref: '#/components/schemas/SetLocationUpdate' + - $ref: '#/components/schemas/SetPropertiesUpdate' + - $ref: '#/components/schemas/RemovePropertiesUpdate' + - $ref: '#/components/schemas/AddViewVersionUpdate' + - $ref: '#/components/schemas/SetCurrentViewVersionUpdate' + + TableRequirement: + type: object + discriminator: + propertyName: type + mapping: + assert-create: '#/components/schemas/AssertCreate' + assert-table-uuid: '#/components/schemas/AssertTableUUID' + assert-ref-snapshot-id: '#/components/schemas/AssertRefSnapshotId' + assert-last-assigned-field-id: '#/components/schemas/AssertLastAssignedFieldId' + assert-current-schema-id: '#/components/schemas/AssertCurrentSchemaId' + assert-last-assigned-partition-id: '#/components/schemas/AssertLastAssignedPartitionId' + assert-default-spec-id: '#/components/schemas/AssertDefaultSpecId' + assert-default-sort-order-id: '#/components/schemas/AssertDefaultSortOrderId' + properties: + type: + type: string + required: + - type + + AssertCreate: + type: object + description: The table must not already exist; used for create transactions + allOf: + - $ref: '#/components/schemas/TableRequirement' + required: + - type + properties: + type: + type: string + const: "assert-create" + + AssertTableUUID: + description: The table UUID must match the requirement's `uuid` + allOf: + - $ref: '#/components/schemas/TableRequirement' + required: + - type + - uuid + properties: + type: + type: string + const: "assert-table-uuid" + uuid: + type: string + + AssertRefSnapshotId: + description: | + The table branch or tag identified by the requirement's `ref` must reference the requirement's `snapshot-id`. + The `snapshot-id` field is required in this object, but in the case of a `null` + the ref must not already exist. + allOf: + - $ref: '#/components/schemas/TableRequirement' + required: + - ref + - snapshot-id + properties: + type: + type: string + const: "assert-ref-snapshot-id" + ref: + type: string + snapshot-id: + type: integer + format: int64 + nullable: true + + AssertLastAssignedFieldId: + description: + The table's last assigned column id must match the requirement's `last-assigned-field-id` + allOf: + - $ref: '#/components/schemas/TableRequirement' + required: + - last-assigned-field-id + properties: + type: + type: string + const: "assert-last-assigned-field-id" + last-assigned-field-id: + type: integer + + AssertCurrentSchemaId: + description: + The table's current schema id must match the requirement's `current-schema-id` + allOf: + - $ref: '#/components/schemas/TableRequirement' + required: + - current-schema-id + properties: + type: + type: string + const: "assert-current-schema-id" + current-schema-id: + type: integer + + AssertLastAssignedPartitionId: + description: + The table's last assigned partition id must match the requirement's `last-assigned-partition-id` + allOf: + - $ref: '#/components/schemas/TableRequirement' + required: + - last-assigned-partition-id + properties: + type: + type: string + const: "assert-last-assigned-partition-id" + last-assigned-partition-id: + type: integer + + AssertDefaultSpecId: + description: + The table's default spec id must match the requirement's `default-spec-id` + allOf: + - $ref: '#/components/schemas/TableRequirement' + required: + - default-spec-id + properties: + type: + type: string + const: "assert-default-spec-id" + default-spec-id: + type: integer + + AssertDefaultSortOrderId: + description: + The table's default sort order id must match the requirement's `default-sort-order-id` + allOf: + - $ref: '#/components/schemas/TableRequirement' + required: + - default-sort-order-id + properties: + type: + type: string + const: "assert-default-sort-order-id" + default-sort-order-id: + type: integer + + ViewRequirement: + type: object + discriminator: + propertyName: type + mapping: + assert-view-uuid: '#/components/schemas/AssertViewUUID' + oneOf: + - $ref: '#/components/schemas/AssertViewUUID' + + AssertViewUUID: + description: The view UUID must match the requirement's `uuid` + required: + - type + - uuid + properties: + type: + type: string + const: "assert-view-uuid" + uuid: + type: string + + StorageCredential: + type: object + required: + - prefix + - config + properties: + prefix: + type: string + description: Indicates a storage location prefix where the credential is relevant. Clients should choose the most + specific prefix (by selecting the longest prefix) if several credentials of the same type are available. + config: + type: object + additionalProperties: + type: string + + LoadCredentialsResponse: + type: object + required: + - storage-credentials + properties: + storage-credentials: + type: array + items: + $ref: '#/components/schemas/StorageCredential' + + LoadTableResult: + description: | + Result used when a table is successfully loaded. + + + The table metadata JSON is returned in the `metadata` field. The corresponding file location of table metadata should be returned in the `metadata-location` field, unless the metadata is not yet committed. For example, a create transaction may return metadata that is staged but not committed. + Clients can check whether metadata has changed by comparing metadata locations after the table has been created. + + + The `config` map returns table-specific configuration for the table's resources, including its HTTP client and FileIO. For example, config may contain a specific FileIO implementation class for the table depending on its underlying storage. + + + The following configurations should be respected by clients: + + ## General Configurations + + - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled + + ## AWS Configurations + + The following configurations should be respected when working with tables stored in AWS S3 + - `client.region`: region to configure client for making requests to AWS + - `s3.access-key-id`: id for credentials that provide access to the data in S3 + - `s3.secret-access-key`: secret for credentials that provide access to data in S3 + - `s3.session-token`: if present, this value should be used for as the session token + - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `s3-signer-open-api.yaml` specification + - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket access is enabled + + ## Storage Credentials + + Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` field. + Clients must first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. + type: object + required: + - metadata + properties: + metadata-location: + type: string + description: May be null if the table is staged as part of a transaction + nullable: true + metadata: + $ref: '#/components/schemas/TableMetadata' + config: + type: object + additionalProperties: + type: string + storage-credentials: + type: array + items: + $ref: '#/components/schemas/StorageCredential' + + ScanTasks: + type: object + description: > + Scan and planning tasks for server-side scan planning + + + - `plan-tasks` contains opaque units of planning work + + - `file-scan-tasks` contains a partial or complete list of table scan tasks + + - `delete-files` contains delete files referenced by file scan tasks + + + Each plan task must be passed to the fetchScanTasks endpoint to fetch + the file scan tasks for the plan task. + + + The list of delete files must contain all delete files referenced by + the file scan tasks. + properties: + delete-files: + description: Delete files referenced by file scan tasks + type: array + items: + $ref: '#/components/schemas/DeleteFile' + file-scan-tasks: + type: array + items: + $ref: '#/components/schemas/FileScanTask' + plan-tasks: + type: array + items: + $ref: '#/components/schemas/PlanTask' + + CompletedPlanningResult: + type: object + description: Completed server-side planning result + allOf: + - $ref: '#/components/schemas/ScanTasks' + - type: object + required: + - status + properties: + status: + $ref: '#/components/schemas/PlanStatus' + const: "completed" + + CompletedPlanningWithIDResult: + type: object + allOf: + - $ref: '#/components/schemas/CompletedPlanningResult' + - type: object + properties: + plan-id: + description: ID used to track a planning request + type: string + + FailedPlanningResult: + type: object + description: Failed server-side planning result + allOf: + - $ref: '#/components/schemas/IcebergErrorResponse' + - type: object + required: + - status + properties: + status: + $ref: '#/components/schemas/PlanStatus' + const: "failed" + + AsyncPlanningResult: + type: object + required: + - status + properties: + status: + $ref: '#/components/schemas/PlanStatus' + const: "submitted" + plan-id: + description: ID used to track a planning request + type: string + + EmptyPlanningResult: + type: object + description: Empty server-side planning result + required: + - status + properties: + status: + $ref: '#/components/schemas/PlanStatus' + enum: ["submitted", "cancelled"] + + PlanStatus: + description: Status of a server-side planning operation + type: string + enum: ["completed", "submitted", "cancelled", "failed"] + + FetchPlanningResult: + type: object + description: Result of server-side scan planning for fetchPlanningResult + discriminator: + propertyName: status + mapping: + completed: '#/components/schemas/CompletedPlanningResult' + submitted: '#/components/schemas/EmptyPlanningResult' + cancelled: '#/components/schemas/EmptyPlanningResult' + failed: '#/components/schemas/FailedPlanningResult' + oneOf: + - $ref: '#/components/schemas/CompletedPlanningResult' + - $ref: '#/components/schemas/FailedPlanningResult' + - $ref: '#/components/schemas/EmptyPlanningResult' + + PlanTableScanResult: + type: object + description: Result of server-side scan planning for planTableScan + discriminator: + propertyName: status + mapping: + completed: '#/components/schemas/CompletedPlanningWithIDResult' + submitted: '#/components/schemas/AsyncPlanningResult' + cancelled: '#/components/schemas/EmptyPlanningResult' + failed: '#/components/schemas/FailedPlanningResult' + oneOf: + - $ref: '#/components/schemas/CompletedPlanningWithIDResult' + - $ref: '#/components/schemas/FailedPlanningResult' + - $ref: '#/components/schemas/AsyncPlanningResult' + - $ref: '#/components/schemas/EmptyPlanningResult' + + FetchScanTasksResult: + type: object + description: Response schema for fetchScanTasks + allOf: + - $ref: '#/components/schemas/ScanTasks' + + CommitTableRequest: + type: object + required: + - requirements + - updates + properties: + identifier: + description: Table identifier to update; must be present for CommitTransactionRequest + $ref: '#/components/schemas/TableIdentifier' + requirements: + type: array + items: + $ref: '#/components/schemas/TableRequirement' + updates: + type: array + items: + $ref: '#/components/schemas/TableUpdate' + + CommitViewRequest: + type: object + required: + - updates + properties: + identifier: + description: View identifier to update + $ref: '#/components/schemas/TableIdentifier' + requirements: + type: array + items: + $ref: '#/components/schemas/ViewRequirement' + updates: + type: array + items: + $ref: '#/components/schemas/ViewUpdate' + + CommitTransactionRequest: + type: object + required: + - table-changes + properties: + table-changes: + type: array + items: + description: Table commit request; must provide an `identifier` + $ref: '#/components/schemas/CommitTableRequest' + + CreateTableRequest: + type: object + required: + - name + - schema + properties: + name: + type: string + location: + type: string + schema: + $ref: '#/components/schemas/Schema' + partition-spec: + $ref: '#/components/schemas/PartitionSpec' + write-order: + $ref: '#/components/schemas/SortOrder' + stage-create: + type: boolean + properties: + type: object + additionalProperties: + type: string + + RegisterTableRequest: + type: object + required: + - name + - metadata-location + properties: + name: + type: string + metadata-location: + type: string + overwrite: + description: Whether to overwrite table metadata if the table already exists + type: boolean + default: false + + CreateViewRequest: + type: object + required: + - name + - schema + - view-version + - properties + properties: + name: + type: string + location: + type: string + schema: + $ref: '#/components/schemas/Schema' + view-version: + $ref: '#/components/schemas/ViewVersion' + description: The view version to create, will replace the schema-id sent within the view-version with the id assigned to the provided schema + properties: + type: object + additionalProperties: + type: string + + LoadViewResult: + description: | + Result used when a view is successfully loaded. + + + The view metadata JSON is returned in the `metadata` field. The corresponding file location of view metadata is returned in the `metadata-location` field. + Clients can check whether metadata has changed by comparing metadata locations after the view has been created. + + The `config` map returns view-specific configuration for the view's resources. + + The following configurations should be respected by clients: + + ## General Configurations + + - `token`: Authorization bearer token to use for view requests if OAuth2 security is enabled + type: object + required: + - metadata-location + - metadata + properties: + metadata-location: + type: string + metadata: + $ref: '#/components/schemas/ViewMetadata' + config: + type: object + additionalProperties: + type: string + + TokenType: + type: string + enum: + - urn:ietf:params:oauth:token-type:access_token + - urn:ietf:params:oauth:token-type:refresh_token + - urn:ietf:params:oauth:token-type:id_token + - urn:ietf:params:oauth:token-type:saml1 + - urn:ietf:params:oauth:token-type:saml2 + - urn:ietf:params:oauth:token-type:jwt + description: + Token type identifier, from RFC 8693 Section 3 + + + See https://datatracker.ietf.org/doc/html/rfc8693#section-3 + + OAuthClientCredentialsRequest: + deprecated: true + description: + The `oauth/tokens` endpoint and related schemas are **DEPRECATED for REMOVAL** from this + spec, see description of the endpoint. + + + OAuth2 client credentials request + + + See https://datatracker.ietf.org/doc/html/rfc6749#section-4.4 + type: object + required: + - grant_type + - client_id + - client_secret + properties: + grant_type: + type: string + enum: + - client_credentials + scope: + type: string + client_id: + type: string + description: + Client ID + + + This can be sent in the request body, but OAuth2 recommends sending it in + a Basic Authorization header. + client_secret: + type: string + description: + Client secret + + + This can be sent in the request body, but OAuth2 recommends sending it in + a Basic Authorization header. + + OAuthTokenExchangeRequest: + deprecated: true + description: + The `oauth/tokens` endpoint and related schemas are **DEPRECATED for REMOVAL** from this + spec, see description of the endpoint. + + + OAuth2 token exchange request + + + See https://datatracker.ietf.org/doc/html/rfc8693 + type: object + required: + - grant_type + - subject_token + - subject_token_type + properties: + grant_type: + type: string + enum: + - urn:ietf:params:oauth:grant-type:token-exchange + scope: + type: string + requested_token_type: + $ref: '#/components/schemas/TokenType' + subject_token: + type: string + description: Subject token for token exchange request + subject_token_type: + $ref: '#/components/schemas/TokenType' + actor_token: + type: string + description: Actor token for token exchange request + actor_token_type: + $ref: '#/components/schemas/TokenType' + + OAuthTokenRequest: + deprecated: true + description: + The `oauth/tokens` endpoint and related schemas are **DEPRECATED for REMOVAL** from this + spec, see description of the endpoint. + anyOf: + - $ref: '#/components/schemas/OAuthClientCredentialsRequest' + - $ref: '#/components/schemas/OAuthTokenExchangeRequest' + + CounterResult: + type: object + required: + - unit + - value + properties: + unit: + type: string + value: + type: integer + format: int64 + + TimerResult: + type: object + required: + - time-unit + - count + - total-duration + properties: + time-unit: + type: string + count: + type: integer + format: int64 + total-duration: + type: integer + format: int64 + + MetricResult: + anyOf: + - $ref: '#/components/schemas/CounterResult' + - $ref: '#/components/schemas/TimerResult' + + Metrics: + type: object + additionalProperties: + $ref: '#/components/schemas/MetricResult' + example: { + "metrics": { + "total-planning-duration": { + "count": 1, + "time-unit": "nanoseconds", + "total-duration": 2644235116 + }, + "result-data-files": { + "unit": "count", + "value": 1, + }, + "result-delete-files": { + "unit": "count", + "value": 0, + }, + "total-data-manifests": { + "unit": "count", + "value": 1, + }, + "total-delete-manifests": { + "unit": "count", + "value": 0, + }, + "scanned-data-manifests": { + "unit": "count", + "value": 1, + }, + "skipped-data-manifests": { + "unit": "count", + "value": 0, + }, + "total-file-size-bytes": { + "unit": "bytes", + "value": 10, + }, + "total-delete-file-size-bytes": { + "unit": "bytes", + "value": 0, + } + } + } + + ReportMetricsRequest: + anyOf: + - $ref: '#/components/schemas/ScanReport' + - $ref: '#/components/schemas/CommitReport' + required: + - report-type + properties: + report-type: + type: string + + ScanReport: + type: object + required: + - table-name + - snapshot-id + - filter + - schema-id + - projected-field-ids + - projected-field-names + - metrics + properties: + table-name: + type: string + snapshot-id: + type: integer + format: int64 + filter: + $ref: '#/components/schemas/Expression' + schema-id: + type: integer + projected-field-ids: + type: array + items: + type: integer + projected-field-names: + type: array + items: + type: string + metrics: + $ref: '#/components/schemas/Metrics' + metadata: + type: object + additionalProperties: + type: string + + CommitReport: + type: object + required: + - table-name + - snapshot-id + - sequence-number + - operation + - metrics + properties: + table-name: + type: string + snapshot-id: + type: integer + format: int64 + sequence-number: + type: integer + format: int64 + operation: + type: string + metrics: + $ref: '#/components/schemas/Metrics' + metadata: + type: object + additionalProperties: + type: string + + OAuthError: + deprecated: true + description: + The `oauth/tokens` endpoint and related schemas are **DEPRECATED for REMOVAL** from this + spec, see description of the endpoint. + type: object + required: + - error + properties: + error: + type: string + enum: + - invalid_request + - invalid_client + - invalid_grant + - unauthorized_client + - unsupported_grant_type + - invalid_scope + error_description: + type: string + error_uri: + type: string + + OAuthTokenResponse: + deprecated: true + description: + The `oauth/tokens` endpoint and related schemas are **DEPRECATED for REMOVAL** from this + spec, see description of the endpoint. + type: object + required: + - access_token + - token_type + properties: + access_token: + type: string + description: + The access token, for client credentials or token exchange + token_type: + type: string + enum: + - bearer + - mac + - N_A + description: + Access token type for client credentials or token exchange + + + See https://datatracker.ietf.org/doc/html/rfc6749#section-7.1 + expires_in: + type: integer + description: + Lifetime of the access token in seconds for client credentials or token exchange + issued_token_type: + $ref: '#/components/schemas/TokenType' + refresh_token: + type: string + description: Refresh token for client credentials or token exchange + scope: + type: string + description: Authorization scope for client credentials or token exchange + + IcebergErrorResponse: + description: JSON wrapper for all error responses (non-2xx) + type: object + required: + - error + properties: + error: + $ref: '#/components/schemas/ErrorModel' + additionalProperties: false + example: + { + "error": { + "message": "The server does not support this operation", + "type": "UnsupportedOperationException", + "code": 406 + } + } + + CreateNamespaceResponse: + type: object + required: + - namespace + properties: + namespace: + $ref: '#/components/schemas/Namespace' + properties: + type: object + additionalProperties: + type: string + description: + Properties stored on the namespace, if supported by the server. + example: { "owner": "Ralph", "created_at": "1452120468" } + default: { } + + GetNamespaceResponse: + type: object + required: + - namespace + properties: + namespace: + $ref: '#/components/schemas/Namespace' + properties: + type: object + description: + Properties stored on the namespace, if supported by the server. + If the server does not support namespace properties, it should return null for this field. + If namespace properties are supported, but none are set, it should return an empty object. + additionalProperties: + type: string + example: { "owner": "Ralph", 'transient_lastDdlTime': '1452120468' } + default: { } + nullable: true + + ListTablesResponse: + type: object + properties: + next-page-token: + $ref: '#/components/schemas/PageToken' + identifiers: + type: array + uniqueItems: true + items: + $ref: '#/components/schemas/TableIdentifier' + + ListNamespacesResponse: + type: object + properties: + next-page-token: + $ref: '#/components/schemas/PageToken' + namespaces: + type: array + uniqueItems: true + items: + $ref: '#/components/schemas/Namespace' + + UpdateNamespacePropertiesResponse: + type: object + required: + - updated + - removed + properties: + updated: + description: List of property keys that were added or updated + type: array + uniqueItems: true + items: + type: string + removed: + description: List of properties that were removed + type: array + items: + type: string + missing: + type: array + items: + type: string + description: + List of properties requested for removal that were not found + in the namespace's properties. Represents a partial success response. + Server's do not need to implement this. + nullable: true + + CommitTableResponse: + type: object + required: + - metadata-location + - metadata + properties: + metadata-location: + type: string + metadata: + $ref: '#/components/schemas/TableMetadata' + + StatisticsFile: + type: object + required: + - snapshot-id + - statistics-path + - file-size-in-bytes + - file-footer-size-in-bytes + - blob-metadata + properties: + snapshot-id: + type: integer + format: int64 + statistics-path: + type: string + file-size-in-bytes: + type: integer + format: int64 + file-footer-size-in-bytes: + type: integer + format: int64 + blob-metadata: + type: array + items: + $ref: '#/components/schemas/BlobMetadata' + + BlobMetadata: + type: object + required: + - type + - snapshot-id + - sequence-number + - fields + properties: + type: + type: string + snapshot-id: + type: integer + format: int64 + sequence-number: + type: integer + format: int64 + fields: + type: array + items: + type: integer + properties: + type: object + additionalProperties: + type: string + + PartitionStatisticsFile: + type: object + required: + - snapshot-id + - statistics-path + - file-size-in-bytes + properties: + snapshot-id: + type: integer + format: int64 + statistics-path: + type: string + file-size-in-bytes: + type: integer + format: int64 + + BooleanTypeValue: + type: boolean + example: true + + IntegerTypeValue: + type: integer + example: 42 + + LongTypeValue: + type: integer + format: int64 + example: 9223372036854775807 + + FloatTypeValue: + type: number + format: float + example: 3.14 + + DoubleTypeValue: + type: number + format: double + example: 123.456 + + DecimalTypeValue: + type: string + description: + "Decimal type values are serialized as strings. Decimals with a positive scale serialize as numeric plain + text, while decimals with a negative scale use scientific notation and the exponent will be equal to the + negated scale. For instance, a decimal with a positive scale is '123.4500', with zero scale is '2', + and with a negative scale is '2E+20'" + example: "123.4500" + + StringTypeValue: + type: string + example: "hello" + + UUIDTypeValue: + type: string + format: uuid + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + maxLength: 36 + minLength: 36 + description: + "UUID type values are serialized as a 36-character lowercase string in standard UUID format as specified + by RFC-4122" + example: "eb26bdb1-a1d8-4aa6-990e-da940875492c" + + DateTypeValue: + type: string + format: date + description: + "Date type values follow the 'YYYY-MM-DD' ISO-8601 standard date format" + example: "2007-12-03" + + TimeTypeValue: + type: string + description: + "Time type values follow the 'HH:MM:SS.ssssss' ISO-8601 format with microsecond precision" + example: "22:31:08.123456" + + TimestampTypeValue: + type: string + description: + "Timestamp type values follow the 'YYYY-MM-DDTHH:MM:SS.ssssss' ISO-8601 format with microsecond precision" + example: "2007-12-03T10:15:30.123456" + + TimestampTzTypeValue: + type: string + description: + "TimestampTz type values follow the 'YYYY-MM-DDTHH:MM:SS.ssssss+00:00' ISO-8601 format with microsecond precision, + and a timezone offset (+00:00 for UTC)" + example: "2007-12-03T10:15:30.123456+00:00" + + TimestampNanoTypeValue: + type: string + description: + "Timestamp_ns type values follow the 'YYYY-MM-DDTHH:MM:SS.sssssssss' ISO-8601 format with nanosecond precision" + example: "2007-12-03T10:15:30.123456789" + + TimestampTzNanoTypeValue: + type: string + description: + "Timestamp_ns type values follow the 'YYYY-MM-DDTHH:MM:SS.sssssssss+00:00' ISO-8601 format with nanosecond + precision, and a timezone offset (+00:00 for UTC)" + example: "2007-12-03T10:15:30.123456789+00:00" + + FixedTypeValue: + type: string + description: + "Fixed length type values are stored and serialized as an uppercase hexadecimal string + preserving the fixed length" + example: "78797A" + + BinaryTypeValue: + type: string + description: + "Binary type values are stored and serialized as an uppercase hexadecimal string" + example: "78797A" + + CountMap: + type: object + properties: + keys: + type: array + items: + $ref: '#/components/schemas/IntegerTypeValue' + description: "List of integer column ids for each corresponding value" + values: + type: array + items: + $ref: '#/components/schemas/LongTypeValue' + description: "List of Long values, matched to 'keys' by index" + example: + { + "keys": [ 1, 2 ], + "values": [ 100,200 ] + } + + ValueMap: + type: object + properties: + keys: + type: array + items: + $ref: '#/components/schemas/IntegerTypeValue' + description: "List of integer column ids for each corresponding value" + values: + type: array + items: + $ref: '#/components/schemas/PrimitiveTypeValue' + description: "List of primitive type values, matched to 'keys' by index" + example: + { + "keys": [ 1, 2 ], + "values": [ 100, "test" ] + } + + PrimitiveTypeValue: + oneOf: + - $ref: '#/components/schemas/BooleanTypeValue' + - $ref: '#/components/schemas/IntegerTypeValue' + - $ref: '#/components/schemas/LongTypeValue' + - $ref: '#/components/schemas/FloatTypeValue' + - $ref: '#/components/schemas/DoubleTypeValue' + - $ref: '#/components/schemas/DecimalTypeValue' + - $ref: '#/components/schemas/StringTypeValue' + - $ref: '#/components/schemas/UUIDTypeValue' + - $ref: '#/components/schemas/DateTypeValue' + - $ref: '#/components/schemas/TimeTypeValue' + - $ref: '#/components/schemas/TimestampTypeValue' + - $ref: '#/components/schemas/TimestampTzTypeValue' + - $ref: '#/components/schemas/TimestampNanoTypeValue' + - $ref: '#/components/schemas/TimestampTzNanoTypeValue' + - $ref: '#/components/schemas/FixedTypeValue' + - $ref: '#/components/schemas/BinaryTypeValue' + + FileFormat: + type: string + enum: + - avro + - orc + - parquet + - puffin + + ContentFile: + discriminator: + propertyName: content + mapping: + data: '#/components/schemas/DataFile' + position-deletes: '#/components/schemas/PositionDeleteFile' + equality-deletes: '#/components/schemas/EqualityDeleteFile' + type: object + required: + - spec-id + - partition + - content + - file-path + - file-format + - file-size-in-bytes + - record-count + properties: + content: + type: string + file-path: + type: string + file-format: + $ref: '#/components/schemas/FileFormat' + spec-id: + type: integer + partition: + type: array + items: + $ref: '#/components/schemas/PrimitiveTypeValue' + description: + A list of partition field values ordered based on the fields of + the partition spec specified by the `spec-id` + example: [1, "bar"] + file-size-in-bytes: + type: integer + format: int64 + description: "Total file size in bytes" + record-count: + type: integer + format: int64 + description: "Number of records in the file" + key-metadata: + allOf: + - $ref: '#/components/schemas/BinaryTypeValue' + description: "Encryption key metadata blob" + split-offsets: + type: array + items: + type: integer + format: int64 + description: "List of splittable offsets" + sort-order-id: + type: integer + + DataFile: + allOf: + - $ref: '#/components/schemas/ContentFile' + type: object + required: + - content + properties: + content: + type: string + const: "data" + first-row-id: + type: integer + format: int64 + description: "The first row ID assigned to the first row in the data file" + column-sizes: + allOf: + - $ref: '#/components/schemas/CountMap' + description: "Map of column id to total count, including null and NaN" + value-counts: + allOf: + - $ref: '#/components/schemas/CountMap' + description: "Map of column id to null value count" + null-value-counts: + allOf: + - $ref: '#/components/schemas/CountMap' + description: "Map of column id to null value count" + nan-value-counts: + allOf: + - $ref: '#/components/schemas/CountMap' + description: "Map of column id to number of NaN values in the column" + lower-bounds: + allOf: + - $ref: '#/components/schemas/ValueMap' + description: "Map of column id to lower bound primitive type values" + upper-bounds: + allOf: + - $ref: '#/components/schemas/ValueMap' + description: "Map of column id to upper bound primitive type values" + + DeleteFile: + discriminator: + propertyName: content + mapping: + position-deletes: '#/components/schemas/PositionDeleteFile' + equality-deletes: '#/components/schemas/EqualityDeleteFile' + oneOf: + - $ref: '#/components/schemas/PositionDeleteFile' + - $ref: '#/components/schemas/EqualityDeleteFile' + + PositionDeleteFile: + allOf: + - $ref: '#/components/schemas/ContentFile' + required: + - content + properties: + content: + type: string + const: "position-deletes" + content-offset: + type: integer + format: int64 + description: Offset within the delete file of delete content + content-size-in-bytes: + type: integer + format: int64 + description: Length, in bytes, of the delete content; required if content-offset is present + + EqualityDeleteFile: + allOf: + - $ref: '#/components/schemas/ContentFile' + required: + - content + properties: + content: + type: string + const: "equality-deletes" + equality-ids: + type: array + items: + type: integer + description: "List of equality field IDs" + + PlanTableScanRequest: + type: object + properties: + snapshot-id: + description: + Identifier for the snapshot to scan in a point-in-time scan + type: integer + format: int64 + select: + description: List of selected schema fields + type: array + items: + $ref: '#/components/schemas/FieldName' + filter: + description: + Expression used to filter the table data + $ref: '#/components/schemas/Expression' + case-sensitive: + description: Enables case sensitive field matching for filter and select + type: boolean + default: true + use-snapshot-schema: + description: + Whether to use the schema at the time the snapshot was written. + + When time travelling, the snapshot schema should be used (true). + When scanning a branch, the table schema should be used (false). + type: boolean + default: false + start-snapshot-id: + description: Starting snapshot ID for an incremental scan (exclusive) + type: integer + format: int64 + end-snapshot-id: + description: + Ending snapshot ID for an incremental scan (inclusive). + + Required when start-snapshot-id is specified. + type: integer + format: int64 + stats-fields: + description: + List of fields for which the service should send column stats. + type: array + items: + $ref: '#/components/schemas/FieldName' + + FieldName: + description: + A full field name (including parent field names), such as those passed + in APIs like Java `Schema#findField(String name)`. + + The nested field name follows these rules + - Nested struct fields are named by concatenating field names at each + struct level using dot (`.`) delimiter, e.g. + employer.contact_info.address.zip_code + - Nested fields in a map key are named using the keyword `key`, e.g. + employee_address_map.key.first_name + - Nested fields in a map value are named using the keyword `value`, + e.g. employee_address_map.value.zip_code + - Nested fields in a list are named using the keyword `element`, e.g. + employees.element.first_name + type: string + + FetchScanTasksRequest: + type: object + required: + - plan-task + properties: + plan-task: + $ref: '#/components/schemas/PlanTask' + + PlanTask: + description: + An opaque string provided by the REST server that represents a + unit of work to produce file scan tasks for scan planning. This allows + clients to fetch tasks across multiple requests to accommodate large result sets. + type: string + + FileScanTask: + type: object + required: + - data-file + properties: + data-file: + $ref: '#/components/schemas/DataFile' + delete-file-references: + description: A list of indices in the delete files array (0-based) + type: array + items: + type: integer + residual-filter: + description: + An optional filter to be applied to rows in this file scan task. + + If the residual is not present, the client must produce the + residual or use the original filter. + allOf: + - $ref: '#/components/schemas/Expression' + + ############################# + # Reusable Response Objects # + ############################# + responses: + + OAuthTokenResponse: + description: OAuth2 token response for client credentials or token exchange + content: + application/json: + schema: + $ref: '#/components/schemas/OAuthTokenResponse' + + OAuthErrorResponse: + description: OAuth2 error response + content: + application/json: + schema: + $ref: '#/components/schemas/OAuthError' + + BadRequestErrorResponse: + description: + Indicates a bad request error. It could be caused by an unexpected request + body format or other forms of request validation failure, such as invalid json. + Usually serves application/json content, although in some cases simple text/plain content might + be returned by the server's middleware. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Malformed request", + "type": "BadRequestException", + "code": 400 + } + } + + # Note that this is a representative example response for use as a shorthand in the spec. + # The fields `message` and `type` as indicated here are not presently prescriptive. + UnauthorizedResponse: + description: + Unauthorized. The REST Catalog SHOULD respond with the 401 UnauthorizedResponse when + the access token provided is expired, revoked, malformed, or invalid for other reasons. + The client MAY request a new access token and retry the request. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Not authorized to make this request", + "type": "NotAuthorizedException", + "code": 401 + } + } + + # Note that this is a representative example response for use as a shorthand in the spec. + # The fields `message` and `type` as indicated here are not presently prescriptive. + ForbiddenResponse: + description: Forbidden. Authenticated user does not have the necessary permissions. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Not authorized to make this request", + "type": "NotAuthorizedException", + "code": 403 + } + } + + # Note that this is a representative example response for use as a shorthand in the spec. + # The fields `message` and `type` as indicated here are not presently prescriptive. + UnsupportedOperationResponse: + description: Not Acceptable / Unsupported Operation. The server does not support this operation. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + example: { + "error": { + "message": "The server does not support this operation", + "type": "UnsupportedOperationException", + "code": 406 + } + } + + IcebergErrorResponse: + description: JSON wrapper for all error responses (non-2xx) + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "The server does not support this operation", + "type": "UnsupportedOperationException", + "code": 406 + } } + + CreateNamespaceResponse: + description: + Represents a successful call to create a namespace. + Returns the namespace created, as well as any properties that were stored for the namespace, + including those the server might have added. Implementations are not required to support namespace + properties. + content: + application/json: + schema: + $ref: '#/components/schemas/CreateNamespaceResponse' + example: { + "namespace": ["accounting", "tax"], + "properties": { "owner": "Ralph", "created_at": "1452120468" } + } + + GetNamespaceResponse: + description: + Returns a namespace, as well as any properties stored on the namespace if namespace properties + are supported by the server. + content: + application/json: + schema: + $ref: '#/components/schemas/GetNamespaceResponse' + + ListTablesResponse: + description: A list of table identifiers + content: + application/json: + schema: + $ref: '#/components/schemas/ListTablesResponse' + examples: + ListTablesResponseNonEmpty: + $ref: '#/components/examples/ListTablesNonEmptyExample' + ListTablesResponseEmpty: + $ref: '#/components/examples/ListTablesEmptyExample' + + ListNamespacesResponse: + description: A list of namespaces + content: + application/json: + schema: + $ref: '#/components/schemas/ListNamespacesResponse' + examples: + NonEmptyResponse: + $ref: '#/components/examples/ListNamespacesNonEmptyExample' + EmptyResponse: + $ref: '#/components/examples/ListNamespacesEmptyExample' + + AuthenticationTimeoutResponse: + description: + This is an optional status response type that the REST Catalog can issue when the + token has expired. The client MAY request a new access token and retry the request. + 401 UnauthorizedResponse SHOULD be preferred over this response type on token expiry. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Credentials have timed out", + "type": "AuthenticationTimeoutException", + "code": 419 + } + } + + ServiceUnavailableResponse: + description: + The service is not ready to handle the request, request could have been partially processed. + + The service may additionally send a Retry-After header to indicate when to retry, a non idempotent + request should only be retried by the client when the Retry-After header is present. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Slow down", + "type": "SlowDownException", + "code": 503 + } + } + + ServerErrorResponse: + description: + A server-side problem that might not be addressable from the client + side. Used for server 5xx errors without more specific documentation in + individual routes. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Internal Server Error", + "type": "InternalServerError", + "code": 500 + } + } + + UpdateNamespacePropertiesResponse: + description: JSON data response for a synchronous update properties request. + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateNamespacePropertiesResponse' + example: { + "updated": [ "owner" ], + "removed": [ "foo" ], + "missing": [ "bar" ] + } + + CreateTableResponse: + description: Table metadata result after creating a table + content: + application/json: + schema: + $ref: '#/components/schemas/LoadTableResult' + headers: + etag: + $ref: '#/components/parameters/etag' + + PlanTableScanResponse: + description: Result of submitting a table scan to plan + content: + application/json: + schema: + $ref: '#/components/schemas/PlanTableScanResult' + + FetchPlanningResultResponse: + description: Result of fetching a submitted scan planning operation + content: + application/json: + schema: + $ref: '#/components/schemas/FetchPlanningResult' + + FetchScanTasksResponse: + description: Result of retrieving additional plan tasks and file scan tasks. + content: + application/json: + schema: + $ref: '#/components/schemas/FetchScanTasksResult' + + LoadTableResponse: + description: Table metadata result when loading a table + content: + application/json: + schema: + $ref: '#/components/schemas/LoadTableResult' + headers: + etag: + $ref: '#/components/parameters/etag' + + LoadViewResponse: + description: View metadata result when loading a view + content: + application/json: + schema: + $ref: '#/components/schemas/LoadViewResult' + + CommitTableResponse: + description: + Response used when a table is successfully updated. + + The table metadata JSON is returned in the metadata field. The corresponding file location of table metadata must be returned in the metadata-location field. Clients can check whether metadata has changed by comparing metadata locations. + content: + application/json: + schema: + $ref: '#/components/schemas/CommitTableResponse' + + LoadCredentialsResponse: + description: Table credentials result when loading credentials for a table + content: + application/json: + schema: + $ref: '#/components/schemas/LoadCredentialsResponse' + + ####################################### + # Common examples of different values # + ####################################### + examples: + + ListTablesEmptyExample: + summary: An empty list for a namespace with no tables + value: { + "identifiers": [ ] + } + + ListNamespacesEmptyExample: + summary: An empty list of namespaces + value: { + "namespaces": [ ] + } + + ListNamespacesNonEmptyExample: + summary: A non-empty list of namespaces + value: { + "namespaces": [ + ["accounting", "tax"], + ["accounting", "credits"] + ] + } + + ListTablesNonEmptyExample: + summary: A non-empty list of table identifiers + value: { + "identifiers": [ + { "namespace": ["accounting", "tax"], "name": "paid" }, + { "namespace": ["accounting", "tax"], "name": "owed" } + ] + } + + MultipartNamespaceAsPathVariable: + summary: A multi-part namespace, as represented in a path parameter + value: "accounting%1Ftax" + + NamespaceAsPathVariable: + summary: A single part namespace, as represented in a path parameter + value: "accounting" + + NamespaceAlreadyExistsError: + summary: The requested namespace already exists + value: { + "error": { + "message": "The given namespace already exists", + "type": "AlreadyExistsException", + "code": 409 + } + } + + NoSuchPlanIdError: + summary: The plan id does not exist + value: { + "error": { + "message": "The plan id does not exist", + "type": "NoSuchPlanIdException", + "code": 404 + } + } + + NoSuchPlanTaskError: + summary: The plan task does not exist + value: { + "error": { + "message": "The plan task does not exist", + "type": "NoSuchPlanTaskException", + "code": 404 + } + } + + NoSuchTableError: + summary: The requested table does not exist + value: { + "error": { + "message": "The given table does not exist", + "type": "NoSuchTableException", + "code": 404 + } + } + + NoSuchViewError: + summary: The requested view does not exist + value: { + "error": { + "message": "The given view does not exist", + "type": "NoSuchViewException", + "code": 404 + } + } + + NoSuchNamespaceError: + summary: The requested namespace does not exist + value: { + "error": { + "message": "The given namespace does not exist", + "type": "NoSuchNamespaceException", + "code": 404 + } + } + + NamespaceNotEmptyError: + summary: The requested namespace is not empty + value: { + "error": { + "message": "The given namespace is not empty", + "type": "NamespaceNotEmptyException", + "code": 409 + } + } + + RenameTableSameNamespace: + summary: Rename a table in the same namespace + value: { + "source": { "namespace": ["accounting", "tax"], "name": "paid" }, + "destination": { "namespace": ["accounting", "tax"], "name": "owed" } + } + + RenameViewSameNamespace: + summary: Rename a view in the same namespace + value: { + "source": { "namespace": [ "accounting", "tax" ], "name": "paid-view" }, + "destination": { "namespace": [ "accounting", "tax" ], "name": "owed-view" } + } + + TableAlreadyExistsError: + summary: The requested table identifier already exists + value: { + "error": { + "message": "The given table already exists", + "type": "AlreadyExistsException", + "code": 409 + } + } + + ViewAlreadyExistsError: + summary: The requested view identifier already exists + value: { + "error": { + "message": "The given view already exists", + "type": "AlreadyExistsException", + "code": 409 + } + } + + # This is an example response and is not meant to be prescriptive regarding the message or type. + UnprocessableEntityDuplicateKey: + summary: + The request body either has the same key multiple times in what should be a map with unique keys + or the request body has keys in two or more fields which should be disjoint sets. + value: { + "error": { + "message": "The request cannot be processed as there is a key present multiple times", + "type": "UnprocessableEntityException", + "code": 422 + } + } + + UpdateAndRemoveNamespacePropertiesRequest: + summary: An update namespace properties request with both properties to remove and properties to upsert. + value: { + "removals": [ "foo", "bar" ], + "updates": { "owner": "Raoul" } + } + + securitySchemes: + OAuth2: + type: oauth2 + description: + This scheme is used for OAuth2 authorization. + + + For unauthorized requests, services should return an appropriate 401 or + 403 response. Implementations must not return altered success (200) + responses when a request is unauthenticated or unauthorized. + + If a separate authorization server is used, substitute the tokenUrl with + the full token path of the external authorization server, and use the + resulting token to access the resources defined in the spec. + flows: + clientCredentials: + tokenUrl: /v1/oauth/tokens + scopes: + catalog: Allows interacting with the Config and Catalog APIs + BearerAuth: + type: http + scheme: bearer