diff --git a/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml b/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml new file mode 100644 index 000000000000..67ac97f93f9c --- /dev/null +++ b/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml @@ -0,0 +1,7 @@ +title: The package management API now has OpenAPI and SolrJ support. +type: changed +authors: + - name: Eric Pugh +links: +- name: PR#4178 + url: https://github.com/apache/solr/pull/4178 diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java new file mode 100644 index 000000000000..cf3a50f8caff --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.QueryParam; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; +import org.apache.solr.client.api.model.PackagesResponse; +import org.apache.solr.client.api.model.SolrJerseyResponse; + +/** V2 API definitions for managing Solr packages. */ +@Path("/cluster/package") +public interface PackageApis { + + @GET + @Operation( + summary = "List all packages registered in this Solr cluster.", + tags = {"package"}) + PackagesResponse listPackages( + @Parameter( + description = + "Name of a package to refresh on the receiving node only (reloads its listeners from ZooKeeper). Used internally for inter-node fan-out by the refresh endpoint; not normally invoked directly.") + @QueryParam("refreshPackage") + String refreshPackage, + @Parameter( + description = + "If provided, the node waits until its package data matches this ZooKeeper version.") + @QueryParam("expectedVersion") + Integer expectedVersion); + + @GET + @Path("/{packageName}") + @Operation( + summary = "Get information about a specific package in this Solr cluster.", + tags = {"package"}) + PackagesResponse getPackage( + @Parameter(description = "The name of the package.", required = true) + @PathParam("packageName") + String packageName, + @Parameter( + description = + "If provided, the node waits until its package data matches this ZooKeeper version before responding.") + @QueryParam("expectedVersion") + Integer expectedVersion); + + @POST + @Path("/{packageName}/versions") + @Operation( + summary = "Add a version of a package to this Solr cluster.", + tags = {"package"}) + SolrJerseyResponse addPackageVersion( + @Parameter(description = "The name of the package.", required = true) + @PathParam("packageName") + String packageName, + @RequestBody(description = "Details of the package version to add.", required = true) + AddPackageVersionRequestBody requestBody); + + @DELETE + @Path("/{packageName}/versions/{version}") + @Operation( + summary = "Delete a specific version of a package from this Solr cluster.", + tags = {"package"}) + SolrJerseyResponse deletePackageVersion( + @Parameter(description = "The name of the package.", required = true) + @PathParam("packageName") + String packageName, + @Parameter(description = "The version of the package to delete.", required = true) + @PathParam("version") + String version); + + @POST + @Path("/{packageName}/refresh") + @Operation( + summary = "Refresh a package on all nodes in this Solr cluster.", + tags = {"package"}) + SolrJerseyResponse refreshPackage( + @Parameter(description = "The name of the package to refresh.", required = true) + @PathParam("packageName") + String packageName); +} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/PackagePayload.java b/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java similarity index 50% rename from solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/PackagePayload.java rename to solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java index 831d87d809f7..3b077c110d3b 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/PackagePayload.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java @@ -14,34 +14,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.solr.client.api.model; -package org.apache.solr.client.solrj.request.beans; - +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; import java.util.List; -import org.apache.solr.common.annotation.JsonProperty; -import org.apache.solr.common.util.ReflectMapWriter; - -/** Just a container class for POJOs used in Package APIs */ -public class PackagePayload { - public static class AddVersion implements ReflectMapWriter { - @JsonProperty(value = "package", required = true) - public String pkg; - @JsonProperty(required = true) - public String version; +/** Request body for adding a version of a package. */ +public class AddPackageVersionRequestBody { - @JsonProperty(required = true) - public List files; + @JsonProperty("version") + @Schema(description = "The version string for this package version.", required = true) + public String version; - @JsonProperty public String manifest; - @JsonProperty public String manifestSHA512; - } + @JsonProperty("files") + @Schema( + description = "File paths from the file store to include in this version.", + required = true) + public List files; - public static class DelVersion implements ReflectMapWriter { - @JsonProperty(value = "package", required = true) - public String pkg; + @JsonProperty("manifest") + @Schema(description = "Optional path to a manifest file in the file store.") + public String manifest; - @JsonProperty(required = true) - public String version; - } + @JsonProperty("manifestSHA512") + @Schema(description = "Optional SHA-512 hash of the manifest file.") + public String manifestSHA512; } diff --git a/solr/api/src/java/org/apache/solr/client/api/model/PackagesResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/PackagesResponse.java new file mode 100644 index 000000000000..0dce425c807b --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/PackagesResponse.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import java.util.List; +import java.util.Map; + +/** Response for the package listing API. */ +public class PackagesResponse extends SolrJerseyResponse { + + @JsonProperty("result") + @Schema(description = "The package data including znode version and package definitions.") + public PackageData result; + + /** Package data returned by the package API. */ + public static class PackageData { + @JsonProperty("znodeVersion") + @Schema(description = "The ZooKeeper version of the packages.json node.") + public int znodeVersion; + + @JsonProperty("packages") + @Schema(description = "Map from package name to list of package versions.") + public Map> packages; + } + + /** Describes a single version of a package. */ + public static class PackageVersion { + @JsonProperty("package") + @Schema(description = "The package name.") + public String pkg; + + @JsonProperty("version") + @Schema(description = "The version string.") + public String version; + + @JsonProperty("files") + @Schema(description = "List of file paths from the file store included in this version.") + public List files; + + @JsonProperty("manifest") + @Schema(description = "Optional manifest reference.") + public String manifest; + + @JsonProperty("manifestSHA512") + @Schema(description = "Optional SHA-512 hash of the manifest.") + public String manifestSHA512; + } +} diff --git a/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java b/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java index 5f6a2e5a0888..e6ec84021081 100644 --- a/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java +++ b/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java @@ -334,7 +334,7 @@ public ApiInfo(PluginMetaHolder infoHolder, List errs) { coreContainer.getPackageLoader().getPackageVersion(pkg, info.version); if (ver.isEmpty()) { // may be we are a bit early. Do a refresh and try again - coreContainer.getPackageLoader().getPackageAPI().refreshPackages(null); + coreContainer.getPackageLoader().getPackageStore().refreshPackages(null); ver = coreContainer.getPackageLoader().getPackageVersion(pkg, info.version); } if (ver.isEmpty()) { diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index a759ce80461a..60aff6f7dac2 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -135,6 +135,7 @@ import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.metrics.otel.OtelUnit; +import org.apache.solr.pkg.ClusterPackage; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequestBase; @@ -847,8 +848,7 @@ private void loadInternal() { registerV2Api(ClusterFileStore.class); packageLoader = new SolrPackageLoader(this); - registerV2Api(packageLoader.getPackageAPI().editAPI); - registerV2Api(packageLoader.getPackageAPI().readAPI); + registerV2Api(ClusterPackage.class); registerV2Api(ZookeeperRead.class); } diff --git a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java index c405fcbcfe95..f19f89f421d7 100644 --- a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java +++ b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java @@ -48,7 +48,7 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; import org.apache.solr.jersey.PermissionName; -import org.apache.solr.pkg.PackageAPI; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.PermissionNameProvider; @@ -87,8 +87,8 @@ public ClusterFileStore( public UploadToFileStoreResponse uploadFile( String filePath, List sig, InputStream requestBody) { final var response = instantiateJerseyResponse(UploadToFileStoreResponse.class); - if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { - throw new RuntimeException(PackageAPI.ERR_MSG); + if (!coreContainer.getPackageLoader().getPackageStore().isEnabled()) { + throw new RuntimeException(PackageStore.ERR_MSG); } try { coreContainer @@ -304,12 +304,12 @@ private void doDelete(String filePath, Boolean localDelete) { @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM) public SolrJerseyResponse deleteFile(String filePath, Boolean localDelete) { final var response = instantiateJerseyResponse(SolrJerseyResponse.class); - if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { - throw new RuntimeException(PackageAPI.ERR_MSG); + if (!coreContainer.getPackageLoader().getPackageStore().isEnabled()) { + throw new RuntimeException(PackageStore.ERR_MSG); } validateName(filePath, true); - if (coreContainer.getPackageLoader().getPackageAPI().isJarInuse(filePath)) { + if (coreContainer.getPackageLoader().getPackageStore().isJarInuse(filePath)) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "jar in use, can't delete"); } doDelete(filePath, localDelete); diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java index e27bfc0e3178..14eb122b9d21 100644 --- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java @@ -87,8 +87,8 @@ import org.apache.solr.handler.admin.api.GetConfigAPI; import org.apache.solr.handler.admin.api.ModifyConfigComponentAPI; import org.apache.solr.handler.admin.api.ModifyParamSetAPI; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.response.SolrQueryResponse; @@ -302,7 +302,7 @@ private void handleGET() { List listeners = req.getCore().getPackageListeners().getListeners(); for (PackageListeners.Listener listener : listeners) { - Map infos = listener.packageDetails(); + Map infos = listener.packageDetails(); if (infos == null || infos.isEmpty()) continue; infos.forEach( (s, mapWriter) -> { diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java index 3897d6c737e0..20711a94b9f8 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java @@ -61,8 +61,8 @@ import org.apache.solr.core.SolrCore; import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.metrics.SolrMetricsContext; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -201,7 +201,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return Map.of(); } diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java index 3f061375d665..4c7868e5aa7f 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java @@ -48,12 +48,10 @@ import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; -import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.FileStoreApi; import org.apache.solr.client.solrj.request.GenericV2SolrRequest; -import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.client.solrj.request.beans.PackagePayload; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.beans.PluginMeta; -import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; @@ -151,20 +149,9 @@ public void uninstall(String packageName, String version) // Delete the package by calling the Package API and remove the Jar printGreen("Executing Package API to remove this package..."); - PackagePayload.DelVersion del = new PackagePayload.DelVersion(); - del.version = version; - del.pkg = packageName; - - V2Request req = - new V2Request.Builder(PackageUtils.PACKAGE_PATH) - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Map.of("delete", del)) - .build(); - try { - V2Response resp = req.process(solrClient); - printGreen("Response: " + resp.jsonStr()); + new PackageApi.DeletePackageVersion(packageName, version).process(solrClient); + printGreen("Package version deleted from Package API."); } catch (SolrServerException | IOException e) { throw new SolrException(ErrorCode.BAD_REQUEST, e); } @@ -176,9 +163,8 @@ public void uninstall(String packageName, String version) String.format(Locale.ROOT, "/package/%s/%s/%s", packageName, version, "manifest.json")); for (String filePath : filesToDelete) { DistribFileStore.deleteZKFileEntry(zkClient, filePath); - String path = "/api/cluster/filestore/files" + filePath; - printGreen("Deleting " + path); - solrClient.request(new GenericSolrRequest(SolrRequest.METHOD.DELETE, path)); + printGreen("Deleting " + filePath); + new FileStoreApi.DeleteFile(filePath).process(solrClient); } printGreen("Package uninstalled: " + packageName + ":" + version + ":-)"); @@ -463,10 +449,7 @@ private Pair, List> deployCollectionPackage( // If updating, refresh the package version for this to take effect if (isUpdate || pegToLatest) { try { - SolrCLI.postJsonToSolr( - solrClient, - PackageUtils.PACKAGE_PATH, - "{\"refresh\": \"" + packageInstance.name + "\"}"); + new PackageApi.RefreshPackage(packageInstance.name).process(solrClient); } catch (Exception ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); } @@ -1075,8 +1058,7 @@ public void undeploy( solrClient, PackageUtils.getCollectionParamsPath(collection), "{set: {PKG_VERSIONS: {" + packageName + ": null}}}"); - SolrCLI.postJsonToSolr( - solrClient, PackageUtils.PACKAGE_PATH, "{\"refresh\": \"" + packageName + "\"}"); + new PackageApi.RefreshPackage(packageName).process(solrClient); } catch (Exception ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); } diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java index 1076f7a3bd3f..6867d17b3653 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java @@ -40,23 +40,17 @@ import org.apache.solr.cli.SolrCLI; import org.apache.solr.client.api.util.SolrVersion; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.FileStoreApi; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.GenericV2SolrRequest; -import org.apache.solr.client.solrj.request.RequestWriter; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.SystemApi; -import org.apache.solr.client.solrj.request.beans.PackagePayload; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Utils; import org.apache.solr.filestore.ClusterFileStore; import org.apache.solr.packagemanager.SolrPackage.Artifact; import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -166,7 +160,7 @@ private String getRepositoriesJson(SolrZkClient zkClient) /** * Install a given package and version from the available repositories to Solr. The various steps * for doing so are, briefly, a) find upload a manifest to package store, b) download the - * artifacts and upload to package store, c) call {@link PackageAPI} to register the package. + * artifacts and upload to package store, c) call {@link PackageManager} to register the package. */ private boolean installPackage(String packageName, String version) throws SolrException { SolrPackageInstance existingPlugin = packageManager.getPackageInstance(packageName, version); @@ -218,10 +212,9 @@ private boolean installPackage(String packageName, String version) throws SolrEx // Call Package API to add this version of the package printGreen("Executing Package API to register this package..."); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = version; - add.pkg = packageName; - add.files = + PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion(packageName); + addRequest.setVersion(version); + addRequest.setFiles( downloaded.stream() .map( file -> @@ -231,21 +224,12 @@ private boolean installPackage(String packageName, String version) throws SolrEx packageName, version, file.getFileName().toString())) - .collect(Collectors.toList()); - add.manifest = "/package/" + packageName + "/" + version + "/manifest.json"; - add.manifestSHA512 = manifestSHA512; - - GenericSolrRequest request = - new GenericV2SolrRequest(SolrRequest.METHOD.POST, PackageUtils.PACKAGE_PATH) { - @Override - public RequestWriter.ContentWriter getContentWriter(String expectedType) { - return new RequestWriter.StringPayloadContentWriter( - "{add:" + add.jsonStr() + "}", "application/json"); - } - }; + .collect(Collectors.toList())); + addRequest.setManifest("/package/" + packageName + "/" + version + "/manifest.json"); + addRequest.setManifestSHA512(manifestSHA512); try { - NamedList resp = solrClient.request(request); - printGreen("Response: " + resp.jsonStr()); + addRequest.process(solrClient); + printGreen("Package version registered successfully."); } catch (SolrServerException | IOException e) { throw new SolrException(ErrorCode.BAD_REQUEST, e); } diff --git a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java new file mode 100644 index 000000000000..e7bdc0979374 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -0,0 +1,334 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.pkg; + +import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; +import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM; +import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM; + +import jakarta.inject.Inject; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import org.apache.solr.api.JerseyResource; +import org.apache.solr.client.api.endpoint.PackageApis; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; +import org.apache.solr.client.api.model.PackagesResponse; +import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.PackageApi; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.Utils; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.filestore.FileStoreUtils; +import org.apache.solr.jersey.PermissionName; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * JAX-RS implementation of the package management API ({@code /api/cluster/package}). + * + * @see PackageApis + */ +public class ClusterPackage extends JerseyResource implements PackageApis { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final int SYNC_MAX_RETRIES = 10; + private static final long SYNC_SLEEP_MS = 10L; + + private final CoreContainer coreContainer; + + @Inject + public ClusterPackage(CoreContainer coreContainer) { + this.coreContainer = coreContainer; + } + + @Override + @PermissionName(PACKAGE_READ_PERM) + public PackagesResponse listPackages(String refreshPackage, Integer expectedVersion) { + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + + if (refreshPackage != null) { + packageStore.packageLoader.notifyListeners(refreshPackage); + return new PackagesResponse(); + } + + if (expectedVersion != null) { + syncToVersion(packageStore, expectedVersion); + } + + final var response = new PackagesResponse(); + response.result = toPackageData(packageStore.pkgs); + return response; + } + + @Override + @PermissionName(PACKAGE_READ_PERM) + public PackagesResponse getPackage(String packageName, Integer expectedVersion) { + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + + if (expectedVersion != null) { + syncToVersion(packageStore, expectedVersion); + } + + final var response = new PackagesResponse(); + response.result = toPackageData(packageStore.pkgs); + // Filter to only the requested package; if absent, return an empty packages map. + if (response.result != null && response.result.packages != null) { + final var pkgVersions = response.result.packages.get(packageName); + response.result.packages = pkgVersions == null ? Map.of() : Map.of(packageName, pkgVersions); + } + return response; + } + + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse addPackageVersion( + String packageName, AddPackageVersionRequestBody requestBody) { + final var response = new SolrJerseyResponse(); + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + + if (!packageStore.isEnabled()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, PackageStore.ERR_MSG); + } + if (requestBody == null || requestBody.files == null || requestBody.files.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No files specified"); + } + + final List errors = new ArrayList<>(); + FileStoreUtils.validateFiles( + coreContainer.getFileStore(), requestBody.files, true, errors::add); + if (!errors.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, String.join("; ", errors)); + } + + final PackageStore.Packages[] finalState = new PackageStore.Packages[1]; + try { + coreContainer + .getZkController() + .getZkClient() + .atomicUpdate( + SOLR_PKGS_PATH, + (stat, bytes) -> { + PackageStore.Packages packages; + try { + packages = + bytes == null + ? new PackageStore.Packages() + : packageStore.mapper.readValue(bytes, PackageStore.Packages.class); + packages = packages.copy(); + } catch (IOException e) { + log.error("Error deserializing packages.json", e); + packages = new PackageStore.Packages(); + } + List list = + packages.packages.computeIfAbsent(packageName, o -> new ArrayList<>()); + for (PackageStore.PkgVersion pkgVersion : list) { + if (Objects.equals(pkgVersion.version, requestBody.version)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Version '" + requestBody.version + "' exists already"); + } + } + list.add(new PackageStore.PkgVersion(packageName, requestBody)); + packages.znodeVersion = stat.getVersion() + 1; + finalState[0] = packages; + return Utils.toJSON(packages); + }); + } catch (KeeperException | InterruptedException e) { + finalState[0] = null; + packageStore.handleZkErr(e); + } + + if (finalState[0] != null) { + packageStore.pkgs = finalState[0]; + notifyAllNodesToSync(packageStore.pkgs.znodeVersion); + coreContainer.getPackageLoader().refreshPackageConf(); + } + + return response; + } + + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse deletePackageVersion(String packageName, String version) { + final var response = new SolrJerseyResponse(); + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + + if (!packageStore.isEnabled()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, PackageStore.ERR_MSG); + } + + try { + coreContainer + .getZkController() + .getZkClient() + .atomicUpdate( + SOLR_PKGS_PATH, + (stat, bytes) -> { + PackageStore.Packages packages; + try { + packages = packageStore.mapper.readValue(bytes, PackageStore.Packages.class); + packages = packages.copy(); + } catch (IOException e) { + packages = new PackageStore.Packages(); + } + + List versions = packages.packages.get(packageName); + if (versions == null || versions.isEmpty()) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No such package: " + packageName); + } + int idxToRemove = -1; + for (int i = 0; i < versions.size(); i++) { + if (Objects.equals(versions.get(i).version, version)) { + idxToRemove = i; + break; + } + } + if (idxToRemove == -1) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No such version: " + version); + } + versions.remove(idxToRemove); + packages.znodeVersion = stat.getVersion() + 1; + return Utils.toJSON(packages); + }); + } catch (KeeperException | InterruptedException e) { + packageStore.handleZkErr(e); + } + + return response; + } + + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse refreshPackage(String packageName) { + final var response = new SolrJerseyResponse(); + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + + SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(packageName); + if (pkg == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No such package: " + packageName); + } + // first refresh on the current node + packageStore.packageLoader.notifyListeners(packageName); + + final var request = new PackageApi.ListPackages(); + request.setRefreshPackage(packageName); + + for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { + final var baseUrl = + coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); + try { + var solrClient = coreContainer.getDefaultHttpSolrClient(); + solrClient.requestWithBaseUrl(baseUrl, request::process); + } catch (SolrServerException | IOException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Failed to refresh package on node: " + liveNode, + e); + } + } + + return response; + } + + private void syncToVersion(PackageStore packageStore, int expectedVersion) { + int origVersion = packageStore.pkgs.znodeVersion; + for (int i = 0; i < SYNC_MAX_RETRIES; i++) { + if (log.isDebugEnabled()) { + log.debug( + "my version is {} , and expected version {}", + packageStore.pkgs.znodeVersion, + expectedVersion); + } + if (packageStore.pkgs.znodeVersion >= expectedVersion) { + if (origVersion < packageStore.pkgs.znodeVersion) { + coreContainer.getPackageLoader().refreshPackageConf(); + } + return; + } + try { + Thread.sleep(SYNC_SLEEP_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + try { + packageStore.pkgs = packageStore.readPkgsFromZk(null, null); + } catch (KeeperException | InterruptedException e) { + packageStore.handleZkErr(e); + } + } + } + + private void notifyAllNodesToSync(int expectedVersion) { + final var request = new PackageApi.ListPackages(); + request.setExpectedVersion(expectedVersion); + + for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { + var baseUrl = coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); + try { + var solrClient = coreContainer.getDefaultHttpSolrClient(); + solrClient.requestWithBaseUrl(baseUrl, request::process); + } catch (SolrServerException | IOException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Failed to notify node: " + + liveNode + + " to sync expected package version: " + + expectedVersion, + e); + } + } + } + + private static PackagesResponse.PackageData toPackageData(PackageStore.Packages packages) { + if (packages == null) { + return null; + } + final var data = new PackagesResponse.PackageData(); + data.znodeVersion = packages.znodeVersion; + data.packages = + packages.packages.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + e -> + e.getValue().stream() + .map(ClusterPackage::toPkgVersionResponse) + .collect(Collectors.toList()))); + return data; + } + + private static PackagesResponse.PackageVersion toPkgVersionResponse( + PackageStore.PkgVersion pkgVersion) { + final var v = new PackagesResponse.PackageVersion(); + v.pkg = pkgVersion.pkg; + v.version = pkgVersion.version; + v.files = pkgVersion.files; + v.manifest = pkgVersion.manifest; + v.manifestSHA512 = pkgVersion.manifestSHA512; + return v; + } +} diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java deleted file mode 100644 index c2d6e0cbeea9..000000000000 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ /dev/null @@ -1,478 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.pkg; - -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; -import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM; -import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM; - -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.io.StringWriter; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import org.apache.solr.api.Command; -import org.apache.solr.api.EndPoint; -import org.apache.solr.api.PayloadObj; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.beans.PackagePayload; -import org.apache.solr.client.solrj.response.JavaBinResponseParser; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.annotation.JsonProperty; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZooKeeperException; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.CommandOperation; -import org.apache.solr.common.util.EnvUtils; -import org.apache.solr.common.util.ReflectMapWriter; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.filestore.FileStoreUtils; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.util.SolrJacksonAnnotationInspector; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.data.Stat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** This implements the public end points (/api/cluster/package) of package API. */ -public class PackageAPI { - public final boolean enablePackages = EnvUtils.getPropertyAsBool("solr.packages.enabled", false); - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static final String ERR_MSG = - "Package loading is not enabled , Start your nodes with -Dsolr.packages.enabled=true"; - - final CoreContainer coreContainer; - private final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); - private final SolrPackageLoader packageLoader; - Packages pkgs; - - public final Edit editAPI = new Edit(); - public final Read readAPI = new Read(); - - public PackageAPI(CoreContainer coreContainer, SolrPackageLoader loader) { - this.coreContainer = coreContainer; - this.packageLoader = loader; - pkgs = new Packages(); - SolrZkClient zkClient = coreContainer.getZkController().getZkClient(); - try { - pkgs = readPkgsFromZk(null, null); - } catch (KeeperException | InterruptedException e) { - pkgs = new Packages(); - // ignore - } - try { - registerListener(zkClient); - } catch (KeeperException | InterruptedException e) { - SolrZkClient.checkInterrupted(e); - } - } - - private void registerListener(SolrZkClient zkClient) - throws KeeperException, InterruptedException { - zkClient.exists( - SOLR_PKGS_PATH, - new Watcher() { - - @Override - public void process(WatchedEvent event) { - // session events are not change events, and do not remove the watcher - if (Event.EventType.None.equals(event.getType())) { - return; - } - synchronized (this) { - log.debug("Updating [{}] ... ", SOLR_PKGS_PATH); - // remake watch - final Watcher thisWatch = this; - refreshPackages(thisWatch); - } - } - }); - } - - public void refreshPackages(Watcher watcher) { - final Stat stat = new Stat(); - try { - final byte[] data = - coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, watcher, stat); - pkgs = readPkgsFromZk(data, stat); - packageLoader.refreshPackageConf(); - } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) { - log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: ", e); - } catch (KeeperException e) { - log.error("A ZK error has occurred", e); - throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); - } catch (InterruptedException e) { - // Restore the interrupted status - Thread.currentThread().interrupt(); - log.warn("Interrupted", e); - } - } - - private Packages readPkgsFromZk(byte[] data, Stat stat) - throws KeeperException, InterruptedException { - - if (data == null || stat == null) { - stat = new Stat(); - data = coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, null, stat); - } - Packages packages = null; - if (data == null || data.length == 0) { - packages = new Packages(); - } else { - try { - packages = mapper.readValue(data, Packages.class); - packages.znodeVersion = stat.getVersion(); - } catch (IOException e) { - // invalid data in packages - // TODO handle properly; - return new Packages(); - } - } - return packages; - } - - public static class Packages implements ReflectMapWriter { - @JsonProperty public int znodeVersion = -1; - - @JsonProperty public Map> packages = new LinkedHashMap<>(); - - public Packages copy() { - Packages p = new Packages(); - p.znodeVersion = this.znodeVersion; - p.packages = new LinkedHashMap<>(); - packages.forEach((s, versions) -> p.packages.put(s, new ArrayList<>(versions))); - return p; - } - } - - public static class PkgVersion implements ReflectMapWriter { - - @JsonProperty("package") - public String pkg; - - @JsonProperty public String version; - - @JsonProperty public List files; - - @JsonProperty public String manifest; - - @JsonProperty public String manifestSHA512; - - public PkgVersion() {} - - public PkgVersion(PackagePayload.AddVersion addVersion) { - this.pkg = addVersion.pkg; - this.version = addVersion.version; - this.files = addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); - this.manifest = addVersion.manifest; - this.manifestSHA512 = addVersion.manifestSHA512; - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof PkgVersion that) { - return Objects.equals(this.version, that.version) && Objects.equals(this.files, that.files); - } - return false; - } - - @Override - public int hashCode() { - return Objects.hash(version); - } - - @Override - public String toString() { - try { - return Utils.writeJson(this, new StringWriter(), false).toString(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - public PkgVersion copy() { - PkgVersion result = new PkgVersion(); - result.pkg = this.pkg; - result.version = this.version; - result.files = this.files; - result.manifest = this.manifest; - result.manifestSHA512 = this.manifestSHA512; - return result; - } - } - - @EndPoint( - method = SolrRequest.METHOD.POST, - path = "/cluster/package", - permission = PACKAGE_EDIT_PERM) - public class Edit { - - @Command(name = "refresh") - public void refresh(PayloadObj payload) { - String p = payload.get(); - if (p == null) { - payload.addError("Package null"); - return; - } - SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(p); - if (pkg == null) { - payload.addError("No such package: " + p); - return; - } - // first refresh my own - packageLoader.notifyListeners(p); - - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("refreshPackage", p); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); - - for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - final var baseUrl = - coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); - try { - var solrClient = coreContainer.getDefaultHttpSolrClient(); - solrClient.requestWithBaseUrl(baseUrl, request::process); - } catch (SolrServerException | IOException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to refresh package on node: " + liveNode, - e); - } - } - } - - @Command(name = "add") - public void add(PayloadObj payload) { - if (!checkEnabled(payload)) return; - PackagePayload.AddVersion add = payload.get(); - if (add.files.isEmpty()) { - payload.addError("No files specified"); - return; - } - FileStoreUtils.validateFiles( - coreContainer.getFileStore(), add.files, true, s -> payload.addError(s)); - if (payload.hasError()) return; - Packages[] finalState = new Packages[1]; - try { - coreContainer - .getZkController() - .getZkClient() - .atomicUpdate( - SOLR_PKGS_PATH, - (stat, bytes) -> { - Packages packages = null; - try { - packages = - bytes == null ? new Packages() : mapper.readValue(bytes, Packages.class); - packages = packages.copy(); - } catch (IOException e) { - log.error("Error deserializing packages.json", e); - packages = new Packages(); - } - List list = - packages.packages.computeIfAbsent(add.pkg, o -> new ArrayList<>()); - for (PkgVersion version : list) { - if (Objects.equals(version.version, add.version)) { - payload.addError("Version '" + add.version + "' exists already"); - return null; - } - } - list.add(new PkgVersion(add)); - packages.znodeVersion = stat.getVersion() + 1; - finalState[0] = packages; - return Utils.toJSON(packages); - }); - } catch (KeeperException | InterruptedException e) { - finalState[0] = null; - handleZkErr(e); - } - if (finalState[0] != null) { - // succeeded in updating - pkgs = finalState[0]; - notifyAllNodesToSync(pkgs.znodeVersion); - packageLoader.refreshPackageConf(); - } - } - - @Command(name = "delete") - public void del(PayloadObj payload) { - if (!checkEnabled(payload)) return; - PackagePayload.DelVersion delVersion = payload.get(); - try { - coreContainer - .getZkController() - .getZkClient() - .atomicUpdate( - SOLR_PKGS_PATH, - (stat, bytes) -> { - Packages packages = null; - try { - packages = mapper.readValue(bytes, Packages.class); - packages = packages.copy(); - } catch (IOException e) { - packages = new Packages(); - } - - List versions = packages.packages.get(delVersion.pkg); - if (versions == null || versions.isEmpty()) { - payload.addError("No such package: " + delVersion.pkg); - return null; // no change - } - int idxToremove = -1; - for (int i = 0; i < versions.size(); i++) { - if (Objects.equals(versions.get(i).version, delVersion.version)) { - idxToremove = i; - break; - } - } - if (idxToremove == -1) { - payload.addError("No such version: " + delVersion.version); - return null; - } - versions.remove(idxToremove); - packages.znodeVersion = stat.getVersion() + 1; - return Utils.toJSON(packages); - }); - } catch (KeeperException | InterruptedException e) { - handleZkErr(e); - } - } - } - - public boolean isEnabled() { - return enablePackages; - } - - private boolean checkEnabled(CommandOperation payload) { - if (!enablePackages) { - payload.addError(ERR_MSG); - return false; - } - return true; - } - - public class Read { - @EndPoint( - method = SolrRequest.METHOD.GET, - path = {"/cluster/package/", "/cluster/package/{name}"}, - permission = PACKAGE_READ_PERM) - public void get(SolrQueryRequest req, SolrQueryResponse rsp) { - String refresh = req.getParams().get("refreshPackage"); - if (refresh != null) { - packageLoader.notifyListeners(refresh); - return; - } - - int expectedVersion = req.getParams().getInt("expectedVersion", -1); - if (expectedVersion != -1) { - syncToVersion(expectedVersion); - } - String name = req.getPathTemplateValues().get("name"); - if (name == null) { - rsp.add("result", pkgs); - } else { - rsp.add("result", Map.of(name, pkgs.packages.get(name))); - } - } - - private void syncToVersion(int expectedVersion) { - int origVersion = pkgs.znodeVersion; - for (int i = 0; i < 10; i++) { - log.debug("my version is {} , and expected version {}", pkgs.znodeVersion, expectedVersion); - if (pkgs.znodeVersion >= expectedVersion) { - if (origVersion < pkgs.znodeVersion) { - packageLoader.refreshPackageConf(); - } - return; - } - try { - Thread.sleep(10); - } catch (InterruptedException e) { - } - try { - pkgs = readPkgsFromZk(null, null); - } catch (KeeperException | InterruptedException e) { - handleZkErr(e); - } - } - } - } - - void notifyAllNodesToSync(int expected) { - - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("expectedVersion", String.valueOf(expected)); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); - - for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - var baseUrl = coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); - try { - var solrClient = coreContainer.getDefaultHttpSolrClient(); - solrClient.requestWithBaseUrl(baseUrl, request::process); - } catch (SolrServerException | IOException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to notify node: " + liveNode + " to sync expected package version: " + expected, - e); - } - } - } - - public void handleZkErr(Exception e) { - log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e)); - } - - public boolean isJarInuse(String path) { - Packages pkg = null; - try { - pkg = readPkgsFromZk(null, null); - } catch (KeeperException.NoNodeException nne) { - return false; - } catch (InterruptedException | KeeperException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } - for (List vers : pkg.packages.values()) { - for (PkgVersion ver : vers) { - if (ver.files.contains(path)) { - return true; - } - } - } - return false; - } -} diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java index 7535bb2c7fe9..b892ae4a9561 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java @@ -110,7 +110,7 @@ public interface Listener { String packageName(); /** fetch the package versions of class names */ - Map packageDetails(); + Map packageDetails(); /** A callback when the package is updated */ void changed(SolrPackageLoader.SolrPackage pkg, Ctx ctx); diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java index 272de544b6e7..8c9fa0a97889 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java @@ -42,7 +42,7 @@ public class PackageListeningClassLoader implements SolrClassLoader, PackageList private final Function pkgVersionSupplier; /** package name and the versions that we are tracking */ - private Map packageVersions = new ConcurrentHashMap<>(1); + private Map packageVersions = new ConcurrentHashMap<>(1); private Map classNameVsPackageName = new ConcurrentHashMap<>(); private final Runnable reloadAction; @@ -99,7 +99,7 @@ public SolrPackageLoader.SolrPackage.Version findPackageVersion( p.getLatest(pkgVersionSupplier.apply(cName.pkg)); if (registerListener) { classNameVsPackageName.put(cName.original, cName.pkg); - PackageAPI.PkgVersion pkgVersion = theVersion.getPkgVersion(); + PackageStore.PkgVersion pkgVersion = theVersion.getPkgVersion(); if (pkgVersion != null) packageVersions.put(cName.pkg, pkgVersion); } return theVersion; @@ -111,7 +111,7 @@ public SolrPackageLoader.SolrPackage.Version findPackageVersion( @Override public MapWriter getPackageVersion(PluginInfo.ClassName cName) { if (cName.pkg == null) return null; - PackageAPI.PkgVersion p = packageVersions.get(cName.pkg); + PackageStore.PkgVersion p = packageVersions.get(cName.pkg); return p == null ? null : p::writeMap; } @@ -162,15 +162,15 @@ public String packageName() { } @Override - public Map packageDetails() { - Map result = new LinkedHashMap<>(); + public Map packageDetails() { + Map result = new LinkedHashMap<>(); classNameVsPackageName.forEach((k, v) -> result.put(k, packageVersions.get(v))); return result; } @Override public void changed(SolrPackageLoader.SolrPackage pkg, Ctx ctx) { - PackageAPI.PkgVersion currVer = packageVersions.get(pkg.name); + PackageStore.PkgVersion currVer = packageVersions.get(pkg.name); if (currVer == null) { // not watching this return; diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java index 419cb033835f..b68f275232fa 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java @@ -58,7 +58,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return Map.of(info.cName.original, pkgVersion.getPkgVersion()); } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageStore.java b/solr/core/src/java/org/apache/solr/pkg/PackageStore.java new file mode 100644 index 000000000000..d51c8f56a2a7 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/pkg/PackageStore.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.pkg; + +import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.io.StringWriter; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.annotation.JsonProperty; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZooKeeperException; +import org.apache.solr.common.util.EnvUtils; +import org.apache.solr.common.util.ReflectMapWriter; +import org.apache.solr.common.util.Utils; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.util.SolrJacksonAnnotationInspector; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Holds package data loaded from ZooKeeper ({@code /solr/packages.json}) and manages ZK watchers. + */ +public class PackageStore { + public final boolean enablePackages = EnvUtils.getPropertyAsBool("solr.packages.enabled", false); + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public static final String ERR_MSG = + "Package loading is not enabled, start your nodes with -Dsolr.packages.enabled=true"; + + final CoreContainer coreContainer; + final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); + final SolrPackageLoader packageLoader; + Packages pkgs; + + public PackageStore(CoreContainer coreContainer, SolrPackageLoader loader) { + this.coreContainer = coreContainer; + this.packageLoader = loader; + pkgs = new Packages(); + SolrZkClient zkClient = coreContainer.getZkController().getZkClient(); + try { + pkgs = readPkgsFromZk(null, null); + } catch (KeeperException | InterruptedException e) { + pkgs = new Packages(); + // ignore + } + try { + registerListener(zkClient); + } catch (KeeperException | InterruptedException e) { + SolrZkClient.checkInterrupted(e); + } + } + + private void registerListener(SolrZkClient zkClient) + throws KeeperException, InterruptedException { + zkClient.exists( + SOLR_PKGS_PATH, + new Watcher() { + + @Override + public void process(WatchedEvent event) { + // session events are not change events, and do not remove the watcher + if (Event.EventType.None.equals(event.getType())) { + return; + } + synchronized (this) { + log.debug("Updating [{}] ... ", SOLR_PKGS_PATH); + // remake watch + final Watcher thisWatch = this; + refreshPackages(thisWatch); + } + } + }); + } + + public void refreshPackages(Watcher watcher) { + final Stat stat = new Stat(); + try { + final byte[] data = + coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, watcher, stat); + pkgs = readPkgsFromZk(data, stat); + packageLoader.refreshPackageConf(); + } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) { + log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: ", e); + } catch (KeeperException e) { + log.error("A ZK error has occurred", e); + throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); + } catch (InterruptedException e) { + // Restore the interrupted status + Thread.currentThread().interrupt(); + log.warn("Interrupted", e); + } + } + + Packages readPkgsFromZk(byte[] data, Stat stat) throws KeeperException, InterruptedException { + + if (data == null || stat == null) { + stat = new Stat(); + data = coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, null, stat); + } + Packages packages; + if (data == null || data.length == 0) { + packages = new Packages(); + } else { + try { + packages = mapper.readValue(data, Packages.class); + packages.znodeVersion = stat.getVersion(); + } catch (IOException e) { + // invalid data in packages + // TODO handle properly; + return new Packages(); + } + } + return packages; + } + + public static class Packages implements ReflectMapWriter { + @JsonProperty public int znodeVersion = -1; + + @JsonProperty public Map> packages = new LinkedHashMap<>(); + + public Packages copy() { + Packages p = new Packages(); + p.znodeVersion = this.znodeVersion; + p.packages = new LinkedHashMap<>(); + packages.forEach((s, versions) -> p.packages.put(s, new ArrayList<>(versions))); + return p; + } + } + + public static class PkgVersion implements ReflectMapWriter { + + @JsonProperty("package") + public String pkg; + + @JsonProperty public String version; + + @JsonProperty public List files; + + @JsonProperty public String manifest; + + @JsonProperty public String manifestSHA512; + + public PkgVersion() {} + + public PkgVersion(String packageName, AddPackageVersionRequestBody addVersion) { + this.pkg = packageName; + this.version = addVersion.version; + this.files = addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); + this.manifest = addVersion.manifest; + this.manifestSHA512 = addVersion.manifestSHA512; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof PkgVersion that) { + return Objects.equals(this.version, that.version) && Objects.equals(this.files, that.files); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(version); + } + + @Override + public String toString() { + try { + return Utils.writeJson(this, new StringWriter(), false).toString(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public PkgVersion copy() { + PkgVersion result = new PkgVersion(); + result.pkg = this.pkg; + result.version = this.version; + result.files = this.files; + result.manifest = this.manifest; + result.manifestSHA512 = this.manifestSHA512; + return result; + } + } + + public boolean isEnabled() { + return enablePackages; + } + + public void handleZkErr(Exception e) { + log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e)); + } + + public boolean isJarInuse(String path) { + Packages pkg; + try { + pkg = readPkgsFromZk(null, null); + } catch (KeeperException.NoNodeException nne) { + return false; + } catch (InterruptedException | KeeperException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } + for (List vers : pkg.packages.values()) { + for (PkgVersion ver : vers) { + if (ver.files.contains(path)) { + return true; + } + } + } + return false; + } +} diff --git a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java index 364bbb919bc0..84fbea486af4 100644 --- a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java @@ -53,9 +53,9 @@ public class SolrPackageLoader implements Closeable { private final CoreContainer coreContainer; private final Map packageClassLoaders = new ConcurrentHashMap<>(); - private PackageAPI.Packages myCopy = new PackageAPI.Packages(); + private PackageStore.Packages myCopy = new PackageStore.Packages(); - private PackageAPI packageAPI; + private PackageStore packageStore; public Optional getPackageVersion(String pkg, String version) { SolrPackage p = packageClassLoaders.get(pkg); @@ -65,12 +65,12 @@ public Optional getPackageVersion(String pkg, String versio public SolrPackageLoader(CoreContainer coreContainer) { this.coreContainer = coreContainer; - packageAPI = new PackageAPI(coreContainer, this); + packageStore = new PackageStore(coreContainer, this); refreshPackageConf(); } - public PackageAPI getPackageAPI() { - return packageAPI; + public PackageStore getPackageStore() { + return packageStore; } public SolrPackage getPackage(String key) { @@ -83,12 +83,12 @@ public Map getPackages() { public void refreshPackageConf() { log.debug( - "{} updated to version {}", ZkStateReader.SOLR_PKGS_PATH, packageAPI.pkgs.znodeVersion); + "{} updated to version {}", ZkStateReader.SOLR_PKGS_PATH, packageStore.pkgs.znodeVersion); List updated = new ArrayList<>(); - Map> modified = getModified(myCopy, packageAPI.pkgs); + Map> modified = getModified(myCopy, packageStore.pkgs); - for (Map.Entry> e : modified.entrySet()) { + for (Map.Entry> e : modified.entrySet()) { if (e.getValue() != null) { SolrPackage p = packageClassLoaders.get(e.getKey()); if (e.getValue() != null && p == null) { @@ -109,14 +109,14 @@ public void refreshPackageConf() { for (SolrCore core : coreContainer.getCores()) { core.getPackageListeners().packagesUpdated(updated); } - myCopy = packageAPI.pkgs; + myCopy = packageStore.pkgs; } - public Map> getModified( - PackageAPI.Packages old, PackageAPI.Packages newPkgs) { - Map> changed = new HashMap<>(); - for (Map.Entry> e : newPkgs.packages.entrySet()) { - List versions = old.packages.get(e.getKey()); + public Map> getModified( + PackageStore.Packages old, PackageStore.Packages newPkgs) { + Map> changed = new HashMap<>(); + for (Map.Entry> e : newPkgs.packages.entrySet()) { + List versions = old.packages.get(e.getKey()); if (versions != null) { if (!Objects.equals(e.getValue(), versions)) { if (log.isInfoEnabled()) { @@ -134,7 +134,7 @@ public Map> getModified( // some packages are deleted altogether for (String s : old.packages.keySet()) { if (!newPkgs.packages.keySet().contains(s)) { - log.info("Package: {} is removed althogether", s); + log.info("Package: {} is removed altogether", s); changed.put(s, null); } } @@ -172,8 +172,8 @@ public Set allVersions() { return myVersions.keySet(); } - private synchronized void updateVersions(List modified) { - for (PackageAPI.PkgVersion v : modified) { + private synchronized void updateVersions(List modified) { + for (PackageStore.PkgVersion v : modified) { Version version = myVersions.get(v.version); if (version == null) { log.info( @@ -194,7 +194,7 @@ private synchronized void updateVersions(List modified) { } Set newVersions = new HashSet<>(); - for (PackageAPI.PkgVersion v : modified) { + for (PackageStore.PkgVersion v : modified) { newVersions.add(v.version); } for (String s : new HashSet<>(myVersions.keySet())) { @@ -258,7 +258,7 @@ public class Version implements MapWriter, Closeable { private final SolrPackage parent; private SolrResourceLoader loader; - private final PackageAPI.PkgVersion version; + private final PackageStore.PkgVersion version; @Override public void writeMap(EntryWriter ew) throws IOException { @@ -266,7 +266,7 @@ public void writeMap(EntryWriter ew) throws IOException { version.writeMap(ew); } - Version(SolrPackage parent, PackageAPI.PkgVersion v) { + Version(SolrPackage parent, PackageStore.PkgVersion v) { this.parent = parent; this.version = v; List paths = new ArrayList<>(); @@ -293,7 +293,7 @@ public String getVersion() { return version.version; } - public PackageAPI.PkgVersion getPkgVersion() { + public PackageStore.PkgVersion getPkgVersion() { return version.copy(); } @@ -330,7 +330,7 @@ static class PackageResourceLoader extends SolrResourceLoader { @Override public boolean addToCoreAware(T obj) { // do not do anything - // this class is not aware of a SolrCore and it is totally not tied to + // this class is not aware of a SolrCore, and it is totally not tied to // the lifecycle of SolrCore. So, this returns 'false' & it should be // taken care of by the caller return false; diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java index 8e7b5ab80ed0..3d0f2fd74e0c 100644 --- a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java +++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java @@ -32,6 +32,7 @@ import java.util.concurrent.Callable; import java.util.function.Predicate; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.solr.client.api.model.UploadToFileStoreResponse; import org.apache.solr.client.solrj.RemoteSolrException; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; @@ -39,12 +40,9 @@ import org.apache.solr.client.solrj.request.FileStoreApi; import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.response.SimpleSolrResponse; -import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.NavigableObject; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.JavaBinCodec; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.SuppressForbidden; @@ -97,7 +95,7 @@ public void testFileStoreManagement() throws Exception { "/package/mypkg/v1.0/runtimelibs.jar", "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - NavigableObject rsp = + UploadToFileStoreResponse rsp = postFile( cluster.getSolrClient(), getFileContent("runtimecode/runtimelibs.jar.bin"), @@ -105,8 +103,7 @@ public void testFileStoreManagement() throws Exception { "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); assertTrue( - Objects.requireNonNullElse(rsp._getStr("message"), "") - .contains("File with same metadata exists ")); + Objects.requireNonNullElse(rsp.message, "").contains("File with same metadata exists ")); assertResponseValues( 10, @@ -344,25 +341,33 @@ public static void uploadKey(byte[] bytes, String path, MiniSolrCloudCluster clu false); } - public static NavigableObject postFile( + public static UploadToFileStoreResponse postFile( SolrClient client, ByteBuffer buffer, String name, String sig) throws SolrServerException, IOException { - String resource = "/cluster/filestore/files" + name; - ModifiableSolrParams params = new ModifiableSolrParams(); - params.add("sig", sig); - V2Response rsp = - new V2Request.Builder(resource) - .withMethod(SolrRequest.METHOD.PUT) - .withPayload(buffer) - .forceV2(true) - .withMimeType("application/octet-stream") - .withParams(params) - .build() - .process(client); - assertEquals(name, rsp.getResponse().get(CommonParams.FILE)); + final var uploadReq = new FileStoreApi.UploadFile(name, new ByteBufferInputStream(buffer)); + if (sig != null) { + uploadReq.setSig(List.of(sig)); + } + final UploadToFileStoreResponse rsp = uploadReq.process(client); + assertEquals(name, rsp.file); return rsp; } + /** + * Upload a file from the test classpath to the distributed file store under the given path, then + * wait until every node in the cluster reports the expected SHA-512 for that path. + */ + public static void postFileAndWait( + MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception { + ByteBuffer fileContent = getFileContent(fname); + @SuppressWarnings("ByteBufferBackingArray") // this is the result of a call to wrap() + String sha512 = DigestUtils.sha512Hex(fileContent.array()); + + postFile(cluster.getSolrClient(), fileContent, path, sig); + + checkAllNodesForFile(cluster, path, Map.of(":files:" + path + ":sha512", sha512), false); + } + /** * Read and return the contents of the file-like resource * diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java index 8f6208449da9..4c35d7b94d7d 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java +++ b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java @@ -41,8 +41,8 @@ import org.apache.solr.client.solrj.RemoteSolrException; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.client.solrj.request.beans.PackagePayload; import org.apache.solr.client.solrj.request.beans.PluginMeta; import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.cloud.ClusterSingleton; @@ -57,10 +57,9 @@ import org.apache.solr.filestore.ClusterFileStore; import org.apache.solr.filestore.TestDistribFileStore; import org.apache.solr.filestore.TestDistribFileStore.Fetcher; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.pkg.SolrPackageLoader; -import org.apache.solr.pkg.TestPackages; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.PermissionNameProvider; @@ -92,7 +91,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return null; // only used to print meta information } @@ -294,12 +293,12 @@ public void testApiFromPackage() throws Exception { byte[] derFile = readFile("cryptokeys/pub_key512.der"); uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); - TestPackages.postFileAndWait( + TestDistribFileStore.postFileAndWait( cluster, "runtimecode/containerplugin.v.1.jar.bin", FILE1, "pmrmWCDafdNpYle2rueAGnU2J6NYlcAey9mkZYbqh+5RdYo2Ln+llLF9voyRj+DDivK9GV1XdtKvD9rgCxlD7Q=="); - TestPackages.postFileAndWait( + TestDistribFileStore.postFileAndWait( cluster, "runtimecode/containerplugin.v.2.jar.bin", FILE2, @@ -308,16 +307,9 @@ public void testApiFromPackage() throws Exception { // We have two versions of the plugin in 2 different jar files. they are already uploaded to // the package store listener.reset(); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = "1.0"; - add.pkg = "mypkg"; - add.files = List.of(FILE1); - V2Request addPkgVersionReq = - new V2Request.Builder("/cluster/package") - .forceV2(forceV2) - .POST() - .withPayload(Map.of("add", add)) - .build(); + PackageApi.AddPackageVersion addPkgVersionReq = new PackageApi.AddPackageVersion("mypkg"); + addPkgVersionReq.setVersion("1.0"); + addPkgVersionReq.setFiles(List.of(FILE1)); addPkgVersionReq.process(cluster.getSolrClient()); assertTrue( "core package listeners did not notify", @@ -335,7 +327,7 @@ public void testApiFromPackage() throws Exception { PluginMeta plugin = new PluginMeta(); plugin.name = "myplugin"; plugin.klass = "mypkg:org.apache.solr.handler.MyPlugin"; - plugin.version = add.version; + plugin.version = "1.0"; final V2Request addPluginReq = postPlugin(Map.of("add", plugin)); addPluginReq.process(cluster.getSolrClient()); version = phaser.awaitAdvanceInterruptibly(version, 10, TimeUnit.SECONDS); @@ -351,12 +343,12 @@ public void testApiFromPackage() throws Exception { TestDistribFileStore.assertResponseValues(invokePlugin, Map.of("/myplugin.version", "1.0")); // now let's upload the jar file for version 2.0 of the plugin - add.version = "2.0"; - add.files = List.of(FILE2); + addPkgVersionReq.setVersion("2.0"); + addPkgVersionReq.setFiles(List.of(FILE2)); addPkgVersionReq.process(cluster.getSolrClient()); // here the plugin version is updated - plugin.version = add.version; + plugin.version = "2.0"; postPlugin(Map.of("update", plugin)).process(cluster.getSolrClient()); version = phaser.awaitAdvanceInterruptibly(version, 10, TimeUnit.SECONDS); diff --git a/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java new file mode 100644 index 000000000000..fef6cdb21be6 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.pkg; + +import static org.apache.solr.filestore.TestDistribFileStore.uploadKey; + +import java.util.List; +import org.apache.solr.client.api.model.PackagesResponse; +import org.apache.solr.client.solrj.RemoteSolrException; +import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; +import org.apache.solr.client.solrj.request.PackageApi; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.filestore.ClusterFileStore; +import org.apache.solr.filestore.TestDistribFileStore; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Integration tests for the JAX-RS-based {@link ClusterPackage}. + * + *

Note: SolrJettyTestRule cannot be used here because the Package API requires ZooKeeper for its + * cluster-level operations. A one-node SolrCloud cluster is used instead. + */ +public class ClusterPackageTest extends SolrCloudTestCase { + + @BeforeClass + public static void setupCluster() throws Exception { + System.setProperty("solr.packages.enabled", "true"); + configureCluster(1) + .withJettyConfig(jetty -> jetty.enableV2(true)) + .addConfig("conf", configset("cloud-minimal")) + .configure(); + } + + @Test + public void testListPackagesReturnsResult() throws Exception { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + PackagesResponse response = new PackageApi.ListPackages().process(client); + assertNotNull("Expected non-null response from GET /cluster/package", response); + assertNotNull("Expected 'result' field in GET /cluster/package response", response.result); + } + } + + @Test + public void testAddAndDeletePackageVersion() throws Exception { + String FILE1 = "/pkgapitestpkg/runtimelibs.jar"; + + // Upload a key and a signed jar file to the filestore + byte[] derFile = TestDistribFileStore.readFile("cryptokeys/pub_key512.der"); + uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); + TestDistribFileStore.postFileAndWait( + cluster, + "runtimecode/runtimelibs.jar.bin", + FILE1, + "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); + + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + // Add a package version via POST /cluster/package/{name}/versions + PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("pkgapitestpkg"); + addRequest.setVersion("1.0"); + addRequest.setFiles(List.of(FILE1)); + addRequest.process(client); + + // Verify the package was added via GET /cluster/package + PackagesResponse listResponse = new PackageApi.ListPackages().process(client); + assertNotNull("Expected non-null list response", listResponse); + assertNotNull("Expected non-null result", listResponse.result); + assertNotNull( + "Expected pkgapitestpkg in packages", listResponse.result.packages.get("pkgapitestpkg")); + assertFalse( + "Expected at least one version", + listResponse.result.packages.get("pkgapitestpkg").isEmpty()); + + // Verify GET /cluster/package/{name} returns only this package + PackagesResponse getByNameResponse = + new PackageApi.GetPackage("pkgapitestpkg").process(client); + assertNotNull("Expected non-null get-by-name response", getByNameResponse); + assertNotNull("Expected non-null result from get-by-name", getByNameResponse.result); + assertNotNull( + "Expected pkgapitestpkg in get-by-name response", + getByNameResponse.result.packages.get("pkgapitestpkg")); + + // Delete the package version via DELETE /cluster/package/{name}/versions/{version} + new PackageApi.DeletePackageVersion("pkgapitestpkg", "1.0").process(client); + + // Verify it's deleted + PackagesResponse listAfterDelete = new PackageApi.ListPackages().process(client); + assertNotNull("Expected non-null list response after delete", listAfterDelete); + assertNotNull("Expected non-null result after delete", listAfterDelete.result); + // After deleting the only version, the package entry should be empty or absent + List versionsAfterDelete = listAfterDelete.result.packages.get("pkgapitestpkg"); + assertTrue( + "Expected no versions after delete", + versionsAfterDelete == null || versionsAfterDelete.isEmpty()); + } + } + + @Test + public void testAddPackageVersionValidatesFiles() { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("testpkg_invalid"); + addRequest.setVersion("1.0"); + addRequest.setFiles(List.of("/nonexistent/file.jar")); + + RemoteSolrException ex = + expectThrows(RemoteSolrException.class, () -> addRequest.process(client)); + assertEquals("Expected 400 for non-existent file", 400, ex.code()); + assertTrue( + "Expected error message to mention the file: " + ex.getMessage(), + ex.getMessage().contains("No such file")); + } + } + + @Test + public void testRefreshNonExistentPackage() { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + RemoteSolrException ex = + expectThrows( + RemoteSolrException.class, + () -> new PackageApi.RefreshPackage("nonexistentpkg_test").process(client)); + assertEquals("Expected 400 for non-existent package", 400, ex.code()); + assertTrue( + "Expected error message to mention the package: " + ex.getMessage(), + ex.getMessage().contains("No such package")); + } + } + + @Test + public void testListPackagesAcceptsRefreshPackageQueryParam() throws Exception { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + PackageApi.ListPackages req = new PackageApi.ListPackages(); + req.setRefreshPackage("nonexistentpkg_refresh"); + // The inter-node refresh signal calls notifyListeners on this node and returns OK even when + // the package doesn't exist; main's behavior is identical. + PackagesResponse response = req.process(client); + assertNotNull(response); + } + } + + @Test + public void testListAndGetPackageAcceptExpectedVersionQueryParam() throws Exception { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + // expectedVersion at-or-below current is a no-op in syncToVersion and must return promptly. + PackageApi.ListPackages listReq = new PackageApi.ListPackages(); + listReq.setExpectedVersion(0); + PackagesResponse listResponse = listReq.process(client); + assertNotNull(listResponse); + assertNotNull(listResponse.result); + + PackageApi.GetPackage getReq = new PackageApi.GetPackage("anything"); + getReq.setExpectedVersion(0); + PackagesResponse getResponse = getReq.process(client); + assertNotNull(getResponse); + assertNotNull(getResponse.result); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java b/solr/core/src/test/org/apache/solr/pkg/ConfigsetPinnedPackageVersionTest.java similarity index 82% rename from solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java rename to solr/core/src/test/org/apache/solr/pkg/ConfigsetPinnedPackageVersionTest.java index 8faac9b68f9e..f5854b6e296f 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/ConfigsetPinnedPackageVersionTest.java @@ -27,11 +27,11 @@ import java.security.Signature; import java.util.Base64; import java.util.List; -import java.util.Map; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.V2Request; +import org.apache.solr.client.solrj.request.FileStoreApi; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.response.SolrResponseBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.filestore.FileStoreAPI; @@ -39,7 +39,12 @@ import org.junit.Before; import org.junit.Test; -public class PackageStoreSchemaPluginsTest extends SolrCloudTestCase { +/** + * Verifies that a configset which pins a specific package version (via {@code params.json}) + * resolves classes from that exact version's jar — even when other versions of the same package are + * also registered. + */ +public class ConfigsetPinnedPackageVersionTest extends SolrCloudTestCase { private static final KeyPair KEY_PAIR; @@ -123,32 +128,17 @@ private static String signature(byte[] content) throws Exception { private void uploadPluginJar(String version, Path jarPath) throws Exception { var pluginRequest = - new V2Request.Builder("/cluster/filestore/files/my-plugin/plugin-" + version + ".jar") - .PUT() - .withParams(params("sig", signature(Files.readAllBytes(jarPath)))) - .withPayload(Files.newInputStream(jarPath)) - .forceV2(true) - .build(); - processRequest(client, pluginRequest); + new FileStoreApi.UploadFile( + "/my-plugin/plugin-" + version + ".jar", Files.newInputStream(jarPath)); + pluginRequest.setSig(List.of(signature(Files.readAllBytes(jarPath)))); + pluginRequest.process(client); } private void registerPackage(String version) throws Exception { - var packageRequest = - new V2Request.Builder("/cluster/package") - .POST() - .forceV2(true) - .withPayload( - Map.of( - "add", - Map.of( - "package", - "mypkg", - "version", - version, - "files", - List.of("/my-plugin/plugin-" + version + ".jar")))) - .build(); - processRequest(client, packageRequest); + var addRequest = new PackageApi.AddPackageVersion("mypkg"); + addRequest.setVersion(version); + addRequest.setFiles(List.of("/my-plugin/plugin-" + version + ".jar")); + addRequest.process(client); } private void createCollection() throws Exception { diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java new file mode 100644 index 000000000000..424cf4fc2f42 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.pkg; + +import static org.apache.solr.filestore.TestDistribFileStore.postFileAndWait; +import static org.apache.solr.filestore.TestDistribFileStore.readFile; +import static org.apache.solr.filestore.TestDistribFileStore.uploadKey; + +import java.util.Arrays; +import java.util.Map; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.PackageApi; +import org.apache.solr.client.solrj.request.V2Request; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.core.SolrCore; +import org.apache.solr.filestore.ClusterFileStore; +import org.apache.solr.filestore.TestDistribFileStore; +import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.IndexSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Verifies that bumping a package version that backs a schema plugin causes the affected core's + * schema to reload and pick up classes from the new package's classloader. + */ +public class PackageSchemaReloadTest extends SolrCloudTestCase { + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + System.setProperty("solr.packages.enabled", "true"); + configureCluster(4) + .withJettyConfig(jetty -> jetty.enableV2(true)) + .addConfig("conf1", configset("schema-package")) + .configure(); + } + + @After + @Override + public void tearDown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + } + super.tearDown(); + } + + @Test + public void testSchemaReloadOnPackageVersionBump() throws Exception { + String COLLECTION_NAME = "testSchemaLoadingColl"; + System.setProperty("managed.schema.mutable", "true"); + + IndexSchema[] schemas = new IndexSchema[2]; // tracks schemas for a selected core + + String FILE1 = "/schemapkg/schema-plugins.jar"; + byte[] derFile = readFile("cryptokeys/pub_key512.der"); + uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); + postFileAndWait( + cluster, + "runtimecode/schema-plugins.jar.bin", + FILE1, + "U+AdO/jgY3DtMpeFRGoTQk72iA5g/qjPvdQYPGBaXB5+ggcTZk4FoIWiueB0bwGJ8Mg3V/elxOqEbD2JR8R0tA=="); + + String FILE2 = "/schemapkg/payload-component.jar"; + postFileAndWait( + cluster, + "runtimecode/payload-component.jar.bin", + FILE2, + "gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q=="); + + // upload package v1.0 + PackageApi.AddPackageVersion addReq = new PackageApi.AddPackageVersion("schemapkg"); + addReq.setVersion("1.0"); + addReq.setFiles(Arrays.asList(FILE1, FILE2)); + addReq.process(cluster.getSolrClient()); + + TestDistribFileStore.assertResponseValues( + 10, + () -> + new V2Request.Builder("/cluster/package") + .withMethod(SolrRequest.METHOD.GET) + .build() + .process(cluster.getSolrClient()), + Map.of( + ":result:packages:schemapkg[0]:version", + "1.0", + ":result:packages:schemapkg[0]:files[0]", + FILE1)); + + CollectionAdminRequest.createCollection(COLLECTION_NAME, "conf1", 2, 2) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); + + // make note of the schema instance for one of the cores + SolrCore.Provider coreProvider = + cluster.getJettySolrRunners().stream() + .flatMap( + jetty -> + jetty.getCoreContainer().getAllCoreNames().stream() + .map(name -> new SolrCore.Provider(jetty.getCoreContainer(), name, null))) + .findFirst() + .orElseThrow(); + + coreProvider.withCore(core -> schemas[0] = core.getLatestSchema()); + + // upload package v2.0 + addReq.setVersion("2.0"); + addReq.setFiles(Arrays.asList(FILE1, FILE2)); + addReq.process(cluster.getSolrClient()); + + TestDistribFileStore.assertResponseValues( + 10, + () -> + new V2Request.Builder("/cluster/package") + .withMethod(SolrRequest.METHOD.GET) + .build() + .process(cluster.getSolrClient()), + Map.of( + ":result:packages:schemapkg[1]:version", + "2.0", + ":result:packages:schemapkg[1]:files[0]", + FILE1)); + + // even though package version 2.0 uses exactly the same files + // as version 1.0, the core schema should still reload, and + // the core should be associated with a different schema instance + TestDistribFileStore.assertResponseValues( + 10, + () -> { + coreProvider.withCore(core -> schemas[1] = core.getLatestSchema()); + return params("schemaReloaded", (schemas[0] != schemas[1]) ? "yes" : "no"); + }, + Map.of("schemaReloaded", "yes")); + + // after the reload, the custom field type class now comes from package v2.0 + String fieldTypeName = "myNewTextFieldWithAnalyzerClass"; + + FieldType fieldTypeV1 = schemas[0].getFieldTypeByName(fieldTypeName); + assertEquals("my.pkg.MyTextField", fieldTypeV1.getClass().getCanonicalName()); + + FieldType fieldTypeV2 = schemas[1].getFieldTypeByName(fieldTypeName); + assertEquals("my.pkg.MyTextField", fieldTypeV2.getClass().getCanonicalName()); + + assertNotEquals( + "my.pkg.MyTextField classes should be from different classloaders", + fieldTypeV1.getClass(), + fieldTypeV2.getClass()); + } +} diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 597e20445441..3dbcb0fef0fa 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -17,11 +17,10 @@ package org.apache.solr.pkg; -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; import static org.apache.solr.common.params.CommonParams.JAVABIN; import static org.apache.solr.common.params.CommonParams.WT; -import static org.apache.solr.core.TestSolrConfigHandler.getFileContent; import static org.apache.solr.filestore.TestDistribFileStore.checkAllNodesForFile; +import static org.apache.solr.filestore.TestDistribFileStore.postFileAndWait; import static org.apache.solr.filestore.TestDistribFileStore.readFile; import static org.apache.solr.filestore.TestDistribFileStore.uploadKey; @@ -30,15 +29,12 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; -import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.Callable; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -import org.apache.commons.codec.digest.DigestUtils; import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory; import org.apache.lucene.analysis.pattern.PatternReplaceCharFilterFactory; import org.apache.lucene.util.ResourceLoader; @@ -49,14 +45,13 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.RequestWriter; import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.client.solrj.request.beans.PackagePayload; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.util.ClientUtils; -import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.NavigableObject; import org.apache.solr.common.SolrInputDocument; @@ -64,7 +59,6 @@ import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.JavaBinCodec; import org.apache.solr.common.util.ReflectMapWriter; import org.apache.solr.common.util.Utils; import org.apache.solr.core.SolrCore; @@ -74,21 +68,32 @@ import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.schema.FieldType; -import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.QParser; import org.apache.solr.search.QParserPlugin; import org.apache.solr.security.AuthorizationContext; import org.apache.solr.util.LogLevel; import org.apache.solr.util.plugin.SolrCoreAware; -import org.apache.zookeeper.data.Stat; import org.eclipse.jetty.client.ContentResponse; import org.eclipse.jetty.client.HttpClient; import org.junit.After; import org.junit.Before; import org.junit.Test; -@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG") +/** + * Multi-node tests covering how Solr consumes packages at runtime: plugin loading via + * {@link SolrPackageLoader} / {@link PackagePluginHolder}, core reload on package version change, + * cross-node refresh fan-out, and package-file download to newly-joined nodes. + * + *

API-contract tests for the {@link ClusterPackage} JAX-RS endpoints live in {@link + * ClusterPackageTest}. Schema-reload-on-version-bump lives in {@link PackageSchemaReloadTest}. + * Configset-pinned-version behavior lives in {@link ConfigsetPinnedPackageVersionTest}. + * + *

Note on the class name: {@code BasePatternReplaceCharFilterFactory} and {@code + * BaseWhitespaceTokenizerFactory} (defined below) are referenced by fully-qualified name from + * pre-compiled test jars in {@code src/test-files/runtimecode/}. Renaming this class would break + * those binary fixtures. + */ +@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.ClusterPackage=DEBUG") public class TestPackages extends SolrCloudTestCase { @Before @@ -132,16 +137,9 @@ public void testCoreReloadingPlugin() throws Exception { FILE1, "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = "1.0"; - add.pkg = "mypkg"; - add.files = Arrays.asList(new String[] {FILE1}); - V2Request req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Map.of("add", add)) - .build(); + PackageApi.AddPackageVersion req = new PackageApi.AddPackageVersion("mypkg"); + req.setVersion("1.0"); + req.setFiles(List.of(FILE1)); req.process(cluster.getSolrClient()); TestDistribFileStore.assertResponseValues( @@ -162,9 +160,9 @@ public void testCoreReloadingPlugin() throws Exception { cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); verifyComponent( - cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", add.pkg, add.version); + cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", "mypkg", "1.0"); - add.version = "2.0"; + req.setVersion("2.0"); req.process(cluster.getSolrClient()); TestDistribFileStore.assertResponseValues( 10, @@ -213,16 +211,9 @@ public void testPluginLoading() throws Exception { EXPR1, "ZOT11arAiPmPZYOHzqodiNnxO9pRyRozWZEBX8XGjU1/HJptFnZK+DI7eXnUtbNaMcbXE2Ze8hh4M/eGyhY8BQ=="); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = "1.0"; - add.pkg = "mypkg"; - add.files = Arrays.asList(new String[] {FILE1, URP1, EXPR1}); - V2Request req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Map.of("add", add)) - .build(); + PackageApi.AddPackageVersion req = new PackageApi.AddPackageVersion("mypkg"); + req.setVersion("1.0"); + req.setFiles(List.of(FILE1, URP1, EXPR1)); req.process(cluster.getSolrClient()); @@ -344,8 +335,8 @@ public void testPluginLoading() throws Exception { URP2, "P/ptFXRvQMd4oKPvadSpd+A9ffwY3gcex5GVFVRy3df0/OF8XT5my8rQz7FZva+2ORbWxdXS8NKwNrbPVHLGXw=="); // add the version using package API - add.version = "1.1"; - add.files = Arrays.asList(new String[] {FILE2, URP2, EXPR1}); + req.setVersion("1.1"); + req.setFiles(List.of(FILE2, URP2, EXPR1)); req.process(cluster.getSolrClient()); verifyComponent( @@ -373,8 +364,8 @@ public void testPluginLoading() throws Exception { FILE3, "a400n4T7FT+2gM0SC6+MfSOExjud8MkhTSFylhvwNjtWwUgKdPFn434Wv7Qc4QEqDVLhQoL3WqYtQmLPti0G4Q=="); - add.version = "2.1"; - add.files = Arrays.asList(new String[] {FILE3, URP2, EXPR1}); + req.setVersion("2.1"); + req.setFiles(List.of(FILE3, URP2, EXPR1)); req.process(cluster.getSolrClient()); // now let's verify that the classes are updated @@ -404,16 +395,7 @@ public void testPluginLoading() throws Exception { assertEquals("Version 2", result.getResults().get(0).getFieldValue("TestVersionedURP.Ver_s")); - PackagePayload.DelVersion delVersion = new PackagePayload.DelVersion(); - delVersion.pkg = "mypkg"; - delVersion.version = "1.0"; - V2Request delete = - new V2Request.Builder("/cluster/package") - .withMethod(SolrRequest.METHOD.POST) - .forceV2(true) - .withPayload(Map.of("delete", delVersion)) - .build(); - delete.process(cluster.getSolrClient()); + new PackageApi.DeletePackageVersion("mypkg", "1.0").process(cluster.getSolrClient()); verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "queryResponseWriter", "json1", "mypkg", "2.1"); @@ -424,9 +406,8 @@ public void testPluginLoading() throws Exception { verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "requestHandler", "/runtime", "mypkg", "2.1"); - // now remove the hughest version. So, it will roll back to the next highest one - delVersion.version = "2.1"; - delete.process(cluster.getSolrClient()); + // now remove the highest version. So, it will roll back to the next highest one + new PackageApi.DeletePackageVersion("mypkg", "2.1").process(cluster.getSolrClient()); verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "queryResponseWriter", "json1", "mypkg", "1.1"); @@ -448,8 +429,8 @@ public RequestWriter.ContentWriter getContentWriter(String expectedType) { } }.setRequiresCollection(true).process(cluster.getSolrClient()); - add.version = "2.1"; - add.files = Arrays.asList(new String[] {FILE3, URP2, EXPR1}); + req.setVersion("2.1"); + req.setFiles(List.of(FILE3, URP2, EXPR1)); req.process(cluster.getSolrClient()); // the collections mypkg is set to use version 1.1 @@ -475,12 +456,7 @@ public RequestWriter.ContentWriter getContentWriter(String expectedType) { // now, let's force every collection using 'mypkg' to refresh // so that it uses version 2.1 - new V2Request.Builder("/cluster/package") - .withMethod(SolrRequest.METHOD.POST) - .withPayload("{refresh : mypkg}") - .forceV2(true) - .build() - .process(cluster.getSolrClient()); + new PackageApi.RefreshPackage("mypkg").process(cluster.getSolrClient()); verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "queryResponseWriter", "json1", "mypkg", "2.1"); @@ -578,130 +554,6 @@ private void verifyComponent( ":config:" + componentType + ":" + componentName + ":_packageinfo_:version", version)); } - @Test - @SuppressWarnings("unchecked") - public void testAPI() throws Exception { - String errPath = "/details[0]/errorMessages[0]"; - String FILE1 = "/mypkg/v.0.12/jar_a.jar"; - String FILE2 = "/mypkg/v.0.12/jar_b.jar"; - String FILE3 = "/mypkg/v.0.13/jar_a.jar"; - - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = "0.12"; - add.pkg = "test_pkg"; - add.files = List.of(FILE1, FILE2); - V2Request req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Map.of("add", add)) - .build(); - - // the files are not yet there. The command should fail with error saying "No such file" - expectError(req, cluster.getSolrClient(), errPath, "No such file:"); - - // post the jar file. No signature is sent - postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1, null); - - add.files = List.of(FILE1); - expectError(req, cluster.getSolrClient(), errPath, FILE1 + " has no signature"); - // now we upload the keys - byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); - // and upload the same file with a different name, but it has proper signature - postFileAndWait( - cluster, - "runtimecode/runtimelibs.jar.bin", - FILE2, - "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - // with correct signature - // after uploading the file, let's delete the keys to see if we get proper error message - add.files = List.of(FILE2); - /*expectError(req, cluster.getSolrClient(), errPath, - "ZooKeeper does not have any public keys");*/ - - // Now lets' put the keys back - - // this time we have a file with proper signature, public keys are in ZK - // so the add {} command should succeed - req.process(cluster.getSolrClient()); - - // Now verify the data in ZK - TestDistribFileStore.assertResponseValues( - 1, - () -> - NavigableObject.wrap( - Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH, null, new Stat()))), - Map.of(":packages:test_pkg[0]:version", "0.12", ":packages:test_pkg[0]:files[0]", FILE2)); - - // post a new jar with a proper signature - postFileAndWait( - cluster, - "runtimecode/runtimelibs_v2.jar.bin", - FILE3, - "j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA=="); - - // this time we are adding the second version of the package (0.13) - add.version = "0.13"; - add.pkg = "test_pkg"; - add.files = List.of(FILE3); - - // this request should succeed - req.process(cluster.getSolrClient()); - // no verify the data (/packages.json) in ZK - TestDistribFileStore.assertResponseValues( - 1, - () -> - NavigableObject.wrap( - Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH, null, new Stat()))), - Map.of(":packages:test_pkg[1]:version", "0.13", ":packages:test_pkg[1]:files[0]", FILE3)); - - // Now we will just delete one version - PackagePayload.DelVersion delVersion = new PackagePayload.DelVersion(); - delVersion.version = "0.1"; // this version does not exist - delVersion.pkg = "test_pkg"; - req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Map.of("delete", delVersion)) - .build(); - - // we are expecting an error - expectError(req, cluster.getSolrClient(), errPath, "No such version:"); - - delVersion.version = "0.12"; // correct version. Should succeed - req.process(cluster.getSolrClient()); - // Verify with ZK that the data is correct - TestDistribFileStore.assertResponseValues( - 1, - () -> - NavigableObject.wrap( - Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH, null, new Stat()))), - Map.of(":packages:test_pkg[0]:version", "0.13", ":packages:test_pkg[0]:files[0]", FILE3)); - - // So far we have been verifying the details with ZK directly - // use the package read API to verify with each node that it has the correct data - for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { - String path = jetty.getBaseURLV2().toString() + "/cluster/package?wt=javabin"; - TestDistribFileStore.assertResponseValues( - 10, - new Callable() { - @Override - public NavigableObject call() throws Exception { - HttpClient solrClient = jetty.getSolrClient().getHttpClient(); - byte[] bytes = solrClient.GET(path).getContent(); - return (NavigableObject) new JavaBinCodec().unmarshal(bytes); - } - }, - Map.of( - ":result:packages:test_pkg[0]:version", - "0.13", - ":result:packages:test_pkg[0]:files[0]", - FILE3)); - } - } - public static class C extends RequestHandlerBase implements SolrCoreAware { static boolean informCalled = false; @@ -739,137 +591,8 @@ public QParser createParser( } } - @Test - public void testSchemaPlugins() throws Exception { - String COLLECTION_NAME = "testSchemaLoadingColl"; - System.setProperty("managed.schema.mutable", "true"); - - IndexSchema[] schemas = new IndexSchema[2]; // tracks schemas for a selected core - - String FILE1 = "/schemapkg/schema-plugins.jar"; - byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); - postFileAndWait( - cluster, - "runtimecode/schema-plugins.jar.bin", - FILE1, - "U+AdO/jgY3DtMpeFRGoTQk72iA5g/qjPvdQYPGBaXB5+ggcTZk4FoIWiueB0bwGJ8Mg3V/elxOqEbD2JR8R0tA=="); - - String FILE2 = "/schemapkg/payload-component.jar"; - postFileAndWait( - cluster, - "runtimecode/payload-component.jar.bin", - FILE2, - "gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q=="); - - // upload package v1.0 - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = "1.0"; - add.pkg = "schemapkg"; - add.files = Arrays.asList(FILE1, FILE2); - V2Request req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Map.of("add", add)) - .build(); - req.process(cluster.getSolrClient()); - - TestDistribFileStore.assertResponseValues( - 10, - () -> - new V2Request.Builder("/cluster/package") - .withMethod(SolrRequest.METHOD.GET) - .build() - .process(cluster.getSolrClient()), - Map.of( - ":result:packages:schemapkg[0]:version", - "1.0", - ":result:packages:schemapkg[0]:files[0]", - FILE1)); - - CollectionAdminRequest.createCollection(COLLECTION_NAME, "conf1", 2, 2) - .process(cluster.getSolrClient()); - cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); - - // make note of the schema instance for one of the cores - SolrCore.Provider coreProvider = - cluster.getJettySolrRunners().stream() - .flatMap( - jetty -> - jetty.getCoreContainer().getAllCoreNames().stream() - .map(name -> new SolrCore.Provider(jetty.getCoreContainer(), name, null))) - .findFirst() - .orElseThrow(); - - coreProvider.withCore(core -> schemas[0] = core.getLatestSchema()); - - // upload package v2.0 - add = new PackagePayload.AddVersion(); - add.version = "2.0"; - add.pkg = "schemapkg"; - add.files = Arrays.asList(FILE1, FILE2); - req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Map.of("add", add)) - .build(); - req.process(cluster.getSolrClient()); - - TestDistribFileStore.assertResponseValues( - 10, - () -> - new V2Request.Builder("/cluster/package") - .withMethod(SolrRequest.METHOD.GET) - .build() - .process(cluster.getSolrClient()), - Map.of( - ":result:packages:schemapkg[1]:version", - "2.0", - ":result:packages:schemapkg[1]:files[0]", - FILE1)); - - // even though package version 2.0 uses exactly the same files - // as version 1.0, the core schema should still reload, and - // the core should be associated with a different schema instance - TestDistribFileStore.assertResponseValues( - 10, - () -> { - coreProvider.withCore(core -> schemas[1] = core.getLatestSchema()); - return params("schemaReloaded", (schemas[0] != schemas[1]) ? "yes" : "no"); - }, - Map.of("schemaReloaded", "yes")); - - // after the reload, the custom field type class now comes from package v2.0 - String fieldTypeName = "myNewTextFieldWithAnalyzerClass"; - - FieldType fieldTypeV1 = schemas[0].getFieldTypeByName(fieldTypeName); - assertEquals("my.pkg.MyTextField", fieldTypeV1.getClass().getCanonicalName()); - - FieldType fieldTypeV2 = schemas[1].getFieldTypeByName(fieldTypeName); - assertEquals("my.pkg.MyTextField", fieldTypeV2.getClass().getCanonicalName()); - - assertNotEquals( - "my.pkg.MyTextField classes should be from different classloaders", - fieldTypeV1.getClass(), - fieldTypeV2.getClass()); - } - - public static void postFileAndWait( - MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception { - ByteBuffer fileContent = getFileContent(fname); - @SuppressWarnings("ByteBufferBackingArray") // this is the result of a call to wrap() - String sha512 = DigestUtils.sha512Hex(fileContent.array()); - - TestDistribFileStore.postFile( - cluster.getSolrClient(), fileContent, path, sig); // has file, but no signature - - TestDistribFileStore.checkAllNodesForFile( - cluster, path, Map.of(":files:" + path + ":sha512", sha512), false); - } - - private void expectError(V2Request req, SolrClient client, String errPath, String expectErrorMsg) + private void expectError( + SolrRequest req, SolrClient client, String errPath, String expectErrorMsg) throws IOException, SolrServerException { try { req.process(client); @@ -899,7 +622,7 @@ public BaseWhitespaceTokenizerFactory(Map args) { } /* - //copy the jav files to a package and then run the main method + //copy the java files to a package and then run the main method public static void main(String[] args) throws Exception { persistZip("/tmp/x.jar", MyPatternReplaceCharFilterFactory.class, MyTextField.class, MyWhitespaceTokenizerFactory.class); }*/ diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc index 323aa9a3bfe3..4814ebb929a4 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc @@ -178,18 +178,19 @@ For example: == API Endpoints -* `GET /api/cluster/package` Get the list of packages -* `POST /api/cluster/package` edit packages -** `add` command: add a version of a package -** `delete` command: delete a version of a package +* `GET /api/cluster/package` Get the list of all packages +* `GET /api/cluster/package/{name}` Get the details of a specific package +* `POST /api/cluster/package/{name}/versions` Add a version of a package +* `DELETE /api/cluster/package/{name}/versions/{version}` Delete a specific version of a package +* `POST /api/cluster/package/{name}/refresh` Refresh a package on all nodes in the cluster === How to Upgrade? -Use the `add` command to add a version that is higher than the current version. +Use the add version endpoint to add a version that is higher than the current version. === How to Downgrade? -Use the `delete` command to delete the highest version and choose the next highest version. +Use the delete version endpoint to delete the highest version and choose the next highest version. === Using Multiple Versions in Parallel @@ -244,11 +245,9 @@ The plugins loaded from packages cannot depend on core level classes. + [source,bash] ---- -curl http://localhost:8983/api/cluster/package -H 'Content-type:application/json' -d ' -{"add": { - "package" : "mypkg", - "version":"1.0", - "files" :["/mypkg/1.0/myplugins.jar"]}}' +curl -X POST http://localhost:8983/api/cluster/package/mypkg/versions \ + -H 'Content-type:application/json' \ + -d '{"version":"1.0","files":["/mypkg/1.0/myplugins.jar"]}' ---- . Verify the created package: @@ -350,11 +349,9 @@ curl http://localhost:8983/api/cluster/filestore/metadata/mypkg/2.0?omitHeader=t + [source,bash] ---- -curl http://localhost:8983/api/cluster/package -H 'Content-type:application/json' -d ' -{"add": { - "package" : "mypkg", - "version":"2.0", - "files" :["/mypkg/2.0/myplugins.jar"]}}' +curl -X POST http://localhost:8983/api/cluster/package/mypkg/versions \ + -H 'Content-type:application/json' \ + -d '{"version":"2.0","files":["/mypkg/2.0/myplugins.jar"]}' ---- . Verify the plugin to see if the correct version of the package is being used: