From a86cf713ad07048a8eb92548e946486901c01829 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Mar 2026 13:42:15 +0000 Subject: [PATCH 01/21] Initial plan From 9422c8c7dd9b9d443cef08fe3c7a49a3624eea39 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Mar 2026 14:08:56 +0000 Subject: [PATCH 02/21] Update TestPackages.java to use new JAX-RS PackageAPI endpoints - Replace PackagePayload.AddVersion with AddPackageVersionRequestBody - Move package name from body to URL: POST /cluster/package/{name}/versions - Replace delete command pattern with DELETE /cluster/package/{name}/versions/{version} - Replace refresh command with POST /cluster/package/{name}/refresh - Update errPath in testAPI from /details[0]/errorMessages[0] to /msg - Remove all add.pkg assignments (pkg now lives in URL path) - Replace add.pkg references in verifyComponent() with literal package name strings - Update import: remove PackagePayload, add AddPackageVersionRequestBody (sorted) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../solr/client/api/endpoint/PackageApis.java | 94 +++++ .../model/AddPackageVersionRequestBody.java | 41 ++ .../client/api/model/PackagesResponse.java | 64 ++++ .../org/apache/solr/core/CoreContainer.java | 4 +- .../java/org/apache/solr/pkg/PackageAPI.java | 253 +------------ .../org/apache/solr/pkg/PackageAPIJaxRs.java | 351 ++++++++++++++++++ .../org/apache/solr/pkg/TestPackages.java | 106 +++--- 7 files changed, 607 insertions(+), 306 deletions(-) create mode 100644 solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java create mode 100644 solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java create mode 100644 solr/api/src/java/org/apache/solr/client/api/model/PackagesResponse.java create mode 100644 solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java new file mode 100644 index 000000000000..9aeb40119d76 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.QueryParam; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; +import org.apache.solr.client.api.model.PackagesResponse; +import org.apache.solr.client.api.model.SolrJerseyResponse; + +/** V2 API definitions for managing Solr packages. */ +@Path("/cluster/package") +public interface PackageApis { + + @GET + @Operation( + summary = "List all packages registered in this Solr cluster.", + tags = {"package"}) + PackagesResponse listPackages( + @Parameter(description = "If provided, the named package is refreshed on this node.") + @QueryParam("refreshPackage") + String refreshPackage, + @Parameter( + description = + "If provided, the node waits until its package data matches this ZooKeeper version.") + @QueryParam("expectedVersion") + Integer expectedVersion); + + @GET + @Path("/{packageName}") + @Operation( + summary = "Get information about a specific package in this Solr cluster.", + tags = {"package"}) + PackagesResponse getPackage( + @Parameter(description = "The name of the package.", required = true) + @PathParam("packageName") + String packageName); + + @POST + @Path("/{packageName}/versions") + @Operation( + summary = "Add a version of a package to this Solr cluster.", + tags = {"package"}) + SolrJerseyResponse addPackageVersion( + @Parameter(description = "The name of the package.", required = true) + @PathParam("packageName") + String packageName, + @RequestBody(description = "Details of the package version to add.", required = true) + AddPackageVersionRequestBody requestBody); + + @DELETE + @Path("/{packageName}/versions/{version}") + @Operation( + summary = "Delete a specific version of a package from this Solr cluster.", + tags = {"package"}) + SolrJerseyResponse deletePackageVersion( + @Parameter(description = "The name of the package.", required = true) + @PathParam("packageName") + String packageName, + @Parameter(description = "The version of the package to delete.", required = true) + @PathParam("version") + String version); + + @POST + @Path("/{packageName}/refresh") + @Operation( + summary = "Refresh a package on all nodes in this Solr cluster.", + tags = {"package"}) + SolrJerseyResponse refreshPackage( + @Parameter(description = "The name of the package to refresh.", required = true) + @PathParam("packageName") + String packageName); +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java new file mode 100644 index 000000000000..1ed868128627 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import java.util.List; + +/** Request body for adding a version of a package. */ +public class AddPackageVersionRequestBody { + + @JsonProperty("version") + @Schema(description = "The version string for this package version.", required = true) + public String version; + + @JsonProperty("files") + @Schema(description = "File paths from the file store to include in this version.", required = true) + public List files; + + @JsonProperty("manifest") + @Schema(description = "Optional path to a manifest file in the file store.") + public String manifest; + + @JsonProperty("manifestSHA512") + @Schema(description = "Optional SHA-512 hash of the manifest file.") + public String manifestSHA512; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/PackagesResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/PackagesResponse.java new file mode 100644 index 000000000000..0dce425c807b --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/PackagesResponse.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import java.util.List; +import java.util.Map; + +/** Response for the package listing API. */ +public class PackagesResponse extends SolrJerseyResponse { + + @JsonProperty("result") + @Schema(description = "The package data including znode version and package definitions.") + public PackageData result; + + /** Package data returned by the package API. */ + public static class PackageData { + @JsonProperty("znodeVersion") + @Schema(description = "The ZooKeeper version of the packages.json node.") + public int znodeVersion; + + @JsonProperty("packages") + @Schema(description = "Map from package name to list of package versions.") + public Map> packages; + } + + /** Describes a single version of a package. */ + public static class PackageVersion { + @JsonProperty("package") + @Schema(description = "The package name.") + public String pkg; + + @JsonProperty("version") + @Schema(description = "The version string.") + public String version; + + @JsonProperty("files") + @Schema(description = "List of file paths from the file store included in this version.") + public List files; + + @JsonProperty("manifest") + @Schema(description = "Optional manifest reference.") + public String manifest; + + @JsonProperty("manifestSHA512") + @Schema(description = "Optional SHA-512 hash of the manifest.") + public String manifestSHA512; + } +} diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index f6bf1dfb36ab..04d7450aa645 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -136,6 +136,7 @@ import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.metrics.otel.OtelUnit; +import org.apache.solr.pkg.PackageAPIJaxRs; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequestBase; @@ -841,8 +842,7 @@ private void loadInternal() { registerV2ApiIfEnabled(ClusterFileStore.class); packageLoader = new SolrPackageLoader(this); - registerV2ApiIfEnabled(packageLoader.getPackageAPI().editAPI); - registerV2ApiIfEnabled(packageLoader.getPackageAPI().readAPI); + registerV2ApiIfEnabled(PackageAPIJaxRs.class); registerV2ApiIfEnabled(ZookeeperRead.class); } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index 24ea22cd3874..f92414b5808e 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -18,8 +18,6 @@ package org.apache.solr.pkg; import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; -import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM; -import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; @@ -31,27 +29,15 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import org.apache.solr.api.Command; -import org.apache.solr.api.EndPoint; -import org.apache.solr.api.PayloadObj; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.beans.PackagePayload; -import org.apache.solr.client.solrj.response.JavaBinResponseParser; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; import org.apache.solr.common.SolrException; import org.apache.solr.common.annotation.JsonProperty; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZooKeeperException; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.CommandOperation; import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.ReflectMapWriter; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; -import org.apache.solr.filestore.FileStoreUtils; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.util.SolrJacksonAnnotationInspector; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; @@ -69,13 +55,10 @@ public class PackageAPI { "Package loading is not enabled , Start your nodes with -Dsolr.packages.enabled=true"; final CoreContainer coreContainer; - private final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); - private final SolrPackageLoader packageLoader; + final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); + final SolrPackageLoader packageLoader; Packages pkgs; - public final Edit editAPI = new Edit(); - public final Read readAPI = new Read(); - public PackageAPI(CoreContainer coreContainer, SolrPackageLoader loader) { this.coreContainer = coreContainer; this.packageLoader = loader; @@ -135,7 +118,7 @@ public void refreshPackages(Watcher watcher) { } } - private Packages readPkgsFromZk(byte[] data, Stat stat) + Packages readPkgsFromZk(byte[] data, Stat stat) throws KeeperException, InterruptedException { if (data == null || stat == null) { @@ -187,10 +170,11 @@ public static class PkgVersion implements ReflectMapWriter { public PkgVersion() {} - public PkgVersion(PackagePayload.AddVersion addVersion) { - this.pkg = addVersion.pkg; + public PkgVersion(String packageName, AddPackageVersionRequestBody addVersion) { + this.pkg = packageName; this.version = addVersion.version; - this.files = addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); + this.files = + addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); this.manifest = addVersion.manifest; this.manifestSHA512 = addVersion.manifestSHA512; } @@ -228,231 +212,10 @@ public PkgVersion copy() { } } - @EndPoint( - method = SolrRequest.METHOD.POST, - path = "/cluster/package", - permission = PACKAGE_EDIT_PERM) - public class Edit { - - @Command(name = "refresh") - public void refresh(PayloadObj payload) { - String p = payload.get(); - if (p == null) { - payload.addError("Package null"); - return; - } - SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(p); - if (pkg == null) { - payload.addError("No such package: " + p); - return; - } - // first refresh my own - packageLoader.notifyListeners(p); - - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("refreshPackage", p); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); - - for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - final var baseUrl = - coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); - try { - var solrClient = coreContainer.getDefaultHttpSolrClient(); - solrClient.requestWithBaseUrl(baseUrl, request::process); - } catch (SolrServerException | IOException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to refresh package on node: " + liveNode, - e); - } - } - } - - @Command(name = "add") - public void add(PayloadObj payload) { - if (!checkEnabled(payload)) return; - PackagePayload.AddVersion add = payload.get(); - if (add.files.isEmpty()) { - payload.addError("No files specified"); - return; - } - FileStoreUtils.validateFiles( - coreContainer.getFileStore(), add.files, true, s -> payload.addError(s)); - if (payload.hasError()) return; - Packages[] finalState = new Packages[1]; - try { - coreContainer - .getZkController() - .getZkClient() - .atomicUpdate( - SOLR_PKGS_PATH, - (stat, bytes) -> { - Packages packages = null; - try { - packages = - bytes == null ? new Packages() : mapper.readValue(bytes, Packages.class); - packages = packages.copy(); - } catch (IOException e) { - log.error("Error deserializing packages.json", e); - packages = new Packages(); - } - List list = - packages.packages.computeIfAbsent(add.pkg, o -> new ArrayList<>()); - for (PkgVersion version : list) { - if (Objects.equals(version.version, add.version)) { - payload.addError("Version '" + add.version + "' exists already"); - return null; - } - } - list.add(new PkgVersion(add)); - packages.znodeVersion = stat.getVersion() + 1; - finalState[0] = packages; - return Utils.toJSON(packages); - }); - } catch (KeeperException | InterruptedException e) { - finalState[0] = null; - handleZkErr(e); - } - if (finalState[0] != null) { - // succeeded in updating - pkgs = finalState[0]; - notifyAllNodesToSync(pkgs.znodeVersion); - packageLoader.refreshPackageConf(); - } - } - - @Command(name = "delete") - public void del(PayloadObj payload) { - if (!checkEnabled(payload)) return; - PackagePayload.DelVersion delVersion = payload.get(); - try { - coreContainer - .getZkController() - .getZkClient() - .atomicUpdate( - SOLR_PKGS_PATH, - (stat, bytes) -> { - Packages packages = null; - try { - packages = mapper.readValue(bytes, Packages.class); - packages = packages.copy(); - } catch (IOException e) { - packages = new Packages(); - } - - List versions = packages.packages.get(delVersion.pkg); - if (versions == null || versions.isEmpty()) { - payload.addError("No such package: " + delVersion.pkg); - return null; // no change - } - int idxToremove = -1; - for (int i = 0; i < versions.size(); i++) { - if (Objects.equals(versions.get(i).version, delVersion.version)) { - idxToremove = i; - break; - } - } - if (idxToremove == -1) { - payload.addError("No such version: " + delVersion.version); - return null; - } - versions.remove(idxToremove); - packages.znodeVersion = stat.getVersion() + 1; - return Utils.toJSON(packages); - }); - } catch (KeeperException | InterruptedException e) { - handleZkErr(e); - } - } - } - public boolean isEnabled() { return enablePackages; } - private boolean checkEnabled(CommandOperation payload) { - if (!enablePackages) { - payload.addError(ERR_MSG); - return false; - } - return true; - } - - public class Read { - @EndPoint( - method = SolrRequest.METHOD.GET, - path = {"/cluster/package/", "/cluster/package/{name}"}, - permission = PACKAGE_READ_PERM) - public void get(SolrQueryRequest req, SolrQueryResponse rsp) { - String refresh = req.getParams().get("refreshPackage"); - if (refresh != null) { - packageLoader.notifyListeners(refresh); - return; - } - - int expectedVersion = req.getParams().getInt("expectedVersion", -1); - if (expectedVersion != -1) { - syncToVersion(expectedVersion); - } - String name = req.getPathTemplateValues().get("name"); - if (name == null) { - rsp.add("result", pkgs); - } else { - rsp.add("result", Collections.singletonMap(name, pkgs.packages.get(name))); - } - } - - private void syncToVersion(int expectedVersion) { - int origVersion = pkgs.znodeVersion; - for (int i = 0; i < 10; i++) { - log.debug("my version is {} , and expected version {}", pkgs.znodeVersion, expectedVersion); - if (pkgs.znodeVersion >= expectedVersion) { - if (origVersion < pkgs.znodeVersion) { - packageLoader.refreshPackageConf(); - } - return; - } - try { - Thread.sleep(10); - } catch (InterruptedException e) { - } - try { - pkgs = readPkgsFromZk(null, null); - } catch (KeeperException | InterruptedException e) { - handleZkErr(e); - } - } - } - } - - void notifyAllNodesToSync(int expected) { - - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("expectedVersion", String.valueOf(expected)); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); - - for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - var baseUrl = coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); - try { - var solrClient = coreContainer.getDefaultHttpSolrClient(); - solrClient.requestWithBaseUrl(baseUrl, request::process); - } catch (SolrServerException | IOException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to notify node: " + liveNode + " to sync expected package version: " + expected, - e); - } - } - } - public void handleZkErr(Exception e) { log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e)); } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java new file mode 100644 index 000000000000..2e2dd0a629bc --- /dev/null +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.pkg; + +import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; +import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM; +import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM; + +import jakarta.inject.Inject; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import org.apache.solr.api.JerseyResource; +import org.apache.solr.client.api.endpoint.PackageApis; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; +import org.apache.solr.client.api.model.PackagesResponse; +import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.response.JavaBinResponseParser; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.Utils; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.filestore.FileStoreUtils; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * JAX-RS implementation of the package management API ({@code /api/cluster/package}). + * + * @see PackageApis + */ +public class PackageAPIJaxRs extends JerseyResource implements PackageApis { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final CoreContainer coreContainer; + private final SolrQueryRequest solrQueryRequest; + private final SolrQueryResponse solrQueryResponse; + + @Inject + public PackageAPIJaxRs( + CoreContainer coreContainer, + SolrQueryRequest solrQueryRequest, + SolrQueryResponse solrQueryResponse) { + this.coreContainer = coreContainer; + this.solrQueryRequest = solrQueryRequest; + this.solrQueryResponse = solrQueryResponse; + } + + @Override + @PermissionName(PACKAGE_READ_PERM) + public PackagesResponse listPackages(String refreshPackage, Integer expectedVersion) { + PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); + + if (refreshPackage != null) { + packageAPI.packageLoader.notifyListeners(refreshPackage); + return instantiateJerseyResponse(PackagesResponse.class); + } + + if (expectedVersion != null) { + syncToVersion(packageAPI, expectedVersion); + } + + final var response = instantiateJerseyResponse(PackagesResponse.class); + response.result = toPackageData(packageAPI.pkgs); + return response; + } + + @Override + @PermissionName(PACKAGE_READ_PERM) + public PackagesResponse getPackage(String packageName) { + PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); + final var response = instantiateJerseyResponse(PackagesResponse.class); + response.result = toPackageData(packageAPI.pkgs); + // Filter to only the requested package + if (response.result != null && response.result.packages != null) { + final var pkgVersions = response.result.packages.get(packageName); + response.result.packages = Collections.singletonMap(packageName, pkgVersions); + } + return response; + } + + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse addPackageVersion( + String packageName, AddPackageVersionRequestBody requestBody) { + final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); + + if (!packageAPI.isEnabled()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, PackageAPI.ERR_MSG); + } + if (requestBody == null || requestBody.files == null || requestBody.files.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No files specified"); + } + + final List errors = new ArrayList<>(); + FileStoreUtils.validateFiles( + coreContainer.getFileStore(), requestBody.files, true, errors::add); + if (!errors.isEmpty()) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, String.join("; ", errors)); + } + + final PackageAPI.Packages[] finalState = new PackageAPI.Packages[1]; + try { + coreContainer + .getZkController() + .getZkClient() + .atomicUpdate( + SOLR_PKGS_PATH, + (stat, bytes) -> { + PackageAPI.Packages packages; + try { + packages = + bytes == null + ? new PackageAPI.Packages() + : packageAPI.mapper.readValue(bytes, PackageAPI.Packages.class); + packages = packages.copy(); + } catch (IOException e) { + log.error("Error deserializing packages.json", e); + packages = new PackageAPI.Packages(); + } + List list = + packages.packages.computeIfAbsent(packageName, o -> new ArrayList<>()); + for (PackageAPI.PkgVersion pkgVersion : list) { + if (Objects.equals(pkgVersion.version, requestBody.version)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Version '" + requestBody.version + "' exists already"); + } + } + list.add(new PackageAPI.PkgVersion(packageName, requestBody)); + packages.znodeVersion = stat.getVersion() + 1; + finalState[0] = packages; + return Utils.toJSON(packages); + }); + } catch (KeeperException | InterruptedException e) { + finalState[0] = null; + packageAPI.handleZkErr(e); + } + + if (finalState[0] != null) { + packageAPI.pkgs = finalState[0]; + notifyAllNodesToSync(packageAPI.pkgs.znodeVersion); + coreContainer.getPackageLoader().refreshPackageConf(); + } + + return response; + } + + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse deletePackageVersion(String packageName, String version) { + final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); + + if (!packageAPI.isEnabled()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, PackageAPI.ERR_MSG); + } + + try { + coreContainer + .getZkController() + .getZkClient() + .atomicUpdate( + SOLR_PKGS_PATH, + (stat, bytes) -> { + PackageAPI.Packages packages; + try { + packages = packageAPI.mapper.readValue(bytes, PackageAPI.Packages.class); + packages = packages.copy(); + } catch (IOException e) { + packages = new PackageAPI.Packages(); + } + + List versions = packages.packages.get(packageName); + if (versions == null || versions.isEmpty()) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No such package: " + packageName); + } + int idxToRemove = -1; + for (int i = 0; i < versions.size(); i++) { + if (Objects.equals(versions.get(i).version, version)) { + idxToRemove = i; + break; + } + } + if (idxToRemove == -1) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No such version: " + version); + } + versions.remove(idxToRemove); + packages.znodeVersion = stat.getVersion() + 1; + return Utils.toJSON(packages); + }); + } catch (KeeperException | InterruptedException e) { + packageAPI.handleZkErr(e); + } + + return response; + } + + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse refreshPackage(String packageName) { + final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); + + SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(packageName); + if (pkg == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No such package: " + packageName); + } + // first refresh on the current node + packageAPI.packageLoader.notifyListeners(packageName); + + final var solrParams = new ModifiableSolrParams(); + solrParams.add("omitHeader", "true"); + solrParams.add("refreshPackage", packageName); + + final var request = + new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); + request.setResponseParser(new JavaBinResponseParser()); + + for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { + final var baseUrl = + coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); + try { + var solrClient = coreContainer.getDefaultHttpSolrClient(); + solrClient.requestWithBaseUrl(baseUrl, request::process); + } catch (SolrServerException | IOException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Failed to refresh package on node: " + liveNode, + e); + } + } + + return response; + } + + private void syncToVersion(PackageAPI packageAPI, int expectedVersion) { + int origVersion = packageAPI.pkgs.znodeVersion; + for (int i = 0; i < 10; i++) { + if (log.isDebugEnabled()) { + log.debug( + "my version is {} , and expected version {}", + packageAPI.pkgs.znodeVersion, + expectedVersion); + } + if (packageAPI.pkgs.znodeVersion >= expectedVersion) { + if (origVersion < packageAPI.pkgs.znodeVersion) { + coreContainer.getPackageLoader().refreshPackageConf(); + } + return; + } + try { + Thread.sleep(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + try { + packageAPI.pkgs = packageAPI.readPkgsFromZk(null, null); + } catch (KeeperException | InterruptedException e) { + packageAPI.handleZkErr(e); + } + } + } + + private void notifyAllNodesToSync(int expectedVersion) { + final var solrParams = new ModifiableSolrParams(); + solrParams.add("omitHeader", "true"); + solrParams.add("expectedVersion", String.valueOf(expectedVersion)); + + final var request = + new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); + request.setResponseParser(new JavaBinResponseParser()); + + for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { + var baseUrl = + coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); + try { + var solrClient = coreContainer.getDefaultHttpSolrClient(); + solrClient.requestWithBaseUrl(baseUrl, request::process); + } catch (SolrServerException | IOException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Failed to notify node: " + + liveNode + + " to sync expected package version: " + + expectedVersion, + e); + } + } + } + + private static PackagesResponse.PackageData toPackageData(PackageAPI.Packages packages) { + if (packages == null) { + return null; + } + final var data = new PackagesResponse.PackageData(); + data.znodeVersion = packages.znodeVersion; + data.packages = + packages.packages.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + e -> + e.getValue().stream() + .map(PackageAPIJaxRs::toPkgVersionResponse) + .collect(Collectors.toList()))); + return data; + } + + private static PackagesResponse.PackageVersion toPkgVersionResponse( + PackageAPI.PkgVersion pkgVersion) { + final var v = new PackagesResponse.PackageVersion(); + v.pkg = pkgVersion.pkg; + v.version = pkgVersion.version; + v.files = pkgVersion.files; + v.manifest = pkgVersion.manifest; + v.manifestSHA512 = pkgVersion.manifestSHA512; + return v; + } +} diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 96d5f86b28a9..0804bbee9a1d 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -42,6 +42,7 @@ import org.apache.lucene.analysis.pattern.PatternReplaceCharFilterFactory; import org.apache.lucene.util.ResourceLoader; import org.apache.lucene.util.ResourceLoaderAware; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; import org.apache.solr.client.solrj.RemoteSolrException; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; @@ -54,7 +55,6 @@ import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.client.solrj.request.beans.PackagePayload; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.cloud.MiniSolrCloudCluster; @@ -130,15 +130,14 @@ public void testCoreReloadingPlugin() throws Exception { FILE1, "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); + AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); add.version = "1.0"; - add.pkg = "mypkg"; add.files = Arrays.asList(new String[] {FILE1}); V2Request req = - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/package/mypkg/versions") .forceV2(true) .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("add", add)) + .withPayload(add) .build(); req.process(cluster.getSolrClient()); @@ -160,7 +159,7 @@ public void testCoreReloadingPlugin() throws Exception { cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); verifyComponent( - cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", add.pkg, add.version); + cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", "mypkg", add.version); add.version = "2.0"; req.process(cluster.getSolrClient()); @@ -211,15 +210,14 @@ public void testPluginLoading() throws Exception { EXPR1, "ZOT11arAiPmPZYOHzqodiNnxO9pRyRozWZEBX8XGjU1/HJptFnZK+DI7eXnUtbNaMcbXE2Ze8hh4M/eGyhY8BQ=="); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); + AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); add.version = "1.0"; - add.pkg = "mypkg"; add.files = Arrays.asList(new String[] {FILE1, URP1, EXPR1}); V2Request req = - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/package/mypkg/versions") .forceV2(true) .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("add", add)) + .withPayload(add) .build(); req.process(cluster.getSolrClient()); @@ -402,16 +400,11 @@ public void testPluginLoading() throws Exception { assertEquals("Version 2", result.getResults().get(0).getFieldValue("TestVersionedURP.Ver_s")); - PackagePayload.DelVersion delVersion = new PackagePayload.DelVersion(); - delVersion.pkg = "mypkg"; - delVersion.version = "1.0"; - V2Request delete = - new V2Request.Builder("/cluster/package") - .withMethod(SolrRequest.METHOD.POST) - .forceV2(true) - .withPayload(Collections.singletonMap("delete", delVersion)) - .build(); - delete.process(cluster.getSolrClient()); + new V2Request.Builder("/cluster/package/mypkg/versions/1.0") + .withMethod(SolrRequest.METHOD.DELETE) + .forceV2(true) + .build() + .process(cluster.getSolrClient()); verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "queryResponseWriter", "json1", "mypkg", "2.1"); @@ -422,9 +415,12 @@ public void testPluginLoading() throws Exception { verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "requestHandler", "/runtime", "mypkg", "2.1"); - // now remove the hughest version. So, it will roll back to the next highest one - delVersion.version = "2.1"; - delete.process(cluster.getSolrClient()); + // now remove the highest version. So, it will roll back to the next highest one + new V2Request.Builder("/cluster/package/mypkg/versions/2.1") + .withMethod(SolrRequest.METHOD.DELETE) + .forceV2(true) + .build() + .process(cluster.getSolrClient()); verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "queryResponseWriter", "json1", "mypkg", "1.1"); @@ -473,9 +469,8 @@ public RequestWriter.ContentWriter getContentWriter(String expectedType) { // now, let's force every collection using 'mypkg' to refresh // so that it uses version 2.1 - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/package/mypkg/refresh") .withMethod(SolrRequest.METHOD.POST) - .withPayload("{refresh : mypkg}") .forceV2(true) .build() .process(cluster.getSolrClient()); @@ -578,20 +573,19 @@ private void verifyComponent( @Test @SuppressWarnings("unchecked") public void testAPI() throws Exception { - String errPath = "/details[0]/errorMessages[0]"; + String errPath = "/msg"; String FILE1 = "/mypkg/v.0.12/jar_a.jar"; String FILE2 = "/mypkg/v.0.12/jar_b.jar"; String FILE3 = "/mypkg/v.0.13/jar_a.jar"; - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); + AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); add.version = "0.12"; - add.pkg = "test_pkg"; add.files = List.of(FILE1, FILE2); V2Request req = - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/package/test_pkg/versions") .forceV2(true) .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("add", add)) + .withPayload(add) .build(); // the files are not yet there. The command should fail with error saying "No such file" @@ -640,7 +634,6 @@ public void testAPI() throws Exception { // this time we are adding the second version of the package (0.13) add.version = "0.13"; - add.pkg = "test_pkg"; add.files = Collections.singletonList(FILE3); // this request should succeed @@ -654,21 +647,20 @@ public void testAPI() throws Exception { Map.of(":packages:test_pkg[1]:version", "0.13", ":packages:test_pkg[1]:files[0]", FILE3)); // Now we will just delete one version - PackagePayload.DelVersion delVersion = new PackagePayload.DelVersion(); - delVersion.version = "0.1"; // this version does not exist - delVersion.pkg = "test_pkg"; - req = - new V2Request.Builder("/cluster/package") + V2Request deleteReq = + new V2Request.Builder("/cluster/package/test_pkg/versions/0.1") .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("delete", delVersion)) + .withMethod(SolrRequest.METHOD.DELETE) .build(); // we are expecting an error - expectError(req, cluster.getSolrClient(), errPath, "No such version:"); + expectError(deleteReq, cluster.getSolrClient(), errPath, "No such version:"); - delVersion.version = "0.12"; // correct version. Should succeed - req.process(cluster.getSolrClient()); + new V2Request.Builder("/cluster/package/test_pkg/versions/0.12") // correct version. Should succeed + .forceV2(true) + .withMethod(SolrRequest.METHOD.DELETE) + .build() + .process(cluster.getSolrClient()); // Verify with ZK that the data is correct TestDistribFileStore.assertResponseValues( 1, @@ -762,17 +754,15 @@ public void testSchemaPlugins() throws Exception { "gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q=="); // upload package v1.0 - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); + AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); add.version = "1.0"; - add.pkg = "schemapkg"; add.files = Arrays.asList(FILE1, FILE2); - V2Request req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("add", add)) - .build(); - req.process(cluster.getSolrClient()); + new V2Request.Builder("/cluster/package/schemapkg/versions") + .forceV2(true) + .withMethod(SolrRequest.METHOD.POST) + .withPayload(add) + .build() + .process(cluster.getSolrClient()); TestDistribFileStore.assertResponseValues( 10, @@ -804,17 +794,15 @@ public void testSchemaPlugins() throws Exception { coreProvider.withCore(core -> schemas[0] = core.getLatestSchema()); // upload package v2.0 - add = new PackagePayload.AddVersion(); + add = new AddPackageVersionRequestBody(); add.version = "2.0"; - add.pkg = "schemapkg"; add.files = Arrays.asList(FILE1, FILE2); - req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("add", add)) - .build(); - req.process(cluster.getSolrClient()); + new V2Request.Builder("/cluster/package/schemapkg/versions") + .forceV2(true) + .withMethod(SolrRequest.METHOD.POST) + .withPayload(add) + .build() + .process(cluster.getSolrClient()); TestDistribFileStore.assertResponseValues( 10, From 744e8ff8a5a23e66277f7b4f07c5439176f6d2db Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Mar 2026 14:14:51 +0000 Subject: [PATCH 03/21] Migrate PackageAPI from @EndPoint/@Command to JAX-RS annotations Co-authored-by: epugh <22395+epugh@users.noreply.github.com> --- .../model/AddPackageVersionRequestBody.java | 4 +- .../java/org/apache/solr/pkg/PackageAPI.java | 6 +- .../org/apache/solr/pkg/PackageAPIJaxRs.java | 6 +- .../apache/solr/pkg/PackageAPIJaxRsTest.java | 164 ++++++++++++++++++ .../org/apache/solr/pkg/TestPackages.java | 3 +- .../pages/package-manager-internals.adoc | 29 ++-- 6 files changed, 186 insertions(+), 26 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java diff --git a/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java index 1ed868128627..3b077c110d3b 100644 --- a/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java @@ -28,7 +28,9 @@ public class AddPackageVersionRequestBody { public String version; @JsonProperty("files") - @Schema(description = "File paths from the file store to include in this version.", required = true) + @Schema( + description = "File paths from the file store to include in this version.", + required = true) public List files; @JsonProperty("manifest") diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index f92414b5808e..d2394aefe917 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -118,8 +118,7 @@ public void refreshPackages(Watcher watcher) { } } - Packages readPkgsFromZk(byte[] data, Stat stat) - throws KeeperException, InterruptedException { + Packages readPkgsFromZk(byte[] data, Stat stat) throws KeeperException, InterruptedException { if (data == null || stat == null) { stat = new Stat(); @@ -173,8 +172,7 @@ public PkgVersion() {} public PkgVersion(String packageName, AddPackageVersionRequestBody addVersion) { this.pkg = packageName; this.version = addVersion.version; - this.files = - addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); + this.files = addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); this.manifest = addVersion.manifest; this.manifestSHA512 = addVersion.manifestSHA512; } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java index 2e2dd0a629bc..9d72b1cff543 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java @@ -123,8 +123,7 @@ public SolrJerseyResponse addPackageVersion( FileStoreUtils.validateFiles( coreContainer.getFileStore(), requestBody.files, true, errors::add); if (!errors.isEmpty()) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, String.join("; ", errors)); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, String.join("; ", errors)); } final PackageAPI.Packages[] finalState = new PackageAPI.Packages[1]; @@ -303,8 +302,7 @@ private void notifyAllNodesToSync(int expectedVersion) { request.setResponseParser(new JavaBinResponseParser()); for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - var baseUrl = - coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); + var baseUrl = coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); try { var solrClient = coreContainer.getDefaultHttpSolrClient(); solrClient.requestWithBaseUrl(baseUrl, request::process); diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java new file mode 100644 index 000000000000..ecfaf7b965b2 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.pkg; + +import static org.apache.solr.filestore.TestDistribFileStore.uploadKey; + +import java.net.URL; +import java.util.List; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.solr.client.solrj.apache.HttpClientUtil; +import org.apache.solr.client.solrj.apache.HttpSolrClient; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.util.Utils; +import org.apache.solr.filestore.ClusterFileStore; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Integration tests for the JAX-RS-based {@link PackageAPIJaxRs}. + * + *

Note: SolrJettyTestRule cannot be used here because the Package API requires ZooKeeper for its + * cluster-level operations. A one-node SolrCloud cluster is used instead. + */ +public class PackageAPIJaxRsTest extends SolrCloudTestCase { + + @BeforeClass + public static void setupCluster() throws Exception { + System.setProperty("solr.packages.enabled", "true"); + configureCluster(1) + .withJettyConfig(jetty -> jetty.enableV2(true)) + .addConfig("conf", configset("cloud-minimal")) + .configure(); + } + + @Test + public void testListPackagesReturnsEmptyResult() throws Exception { + URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); + String packageUrl = + cluster.getJettySolrRunner(0).getBaseURLV2().toString() + "/cluster/package"; + + try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { + Object response = + HttpClientUtil.executeGET(client.getHttpClient(), packageUrl, Utils.JSONCONSUMER); + assertNotNull("Expected non-null response from GET /cluster/package", response); + // The response should have a 'result' field with 'packages' and 'znodeVersion' + Object result = Utils.getObjectByPath(response, true, "result"); + assertNotNull("Expected 'result' field in GET /cluster/package response", result); + } + } + + @Test + public void testAddDeletePackageVersion() throws Exception { + URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); + String baseUrlV2 = cluster.getJettySolrRunner(0).getBaseURLV2().toString(); + String FILE1 = "/testpkg/runtimelibs.jar"; + + // Upload a key and a signed jar file to the filestore + byte[] derFile = + org.apache.solr.filestore.TestDistribFileStore.readFile("cryptokeys/pub_key512.der"); + uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); + org.apache.solr.pkg.TestPackages.postFileAndWait( + cluster, + "runtimecode/runtimelibs.jar.bin", + FILE1, + "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); + + try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { + // Add a package version via POST /cluster/package/{name}/versions + String addUrl = baseUrlV2 + "/cluster/package/testpkg/versions"; + HttpPost httpPost = new HttpPost(addUrl); + httpPost.setHeader("Content-Type", "application/json"); + httpPost.setEntity(new StringEntity("{\"version\":\"1.0\",\"files\":[\"" + FILE1 + "\"]}")); + Object addResponse = + HttpClientUtil.executeHttpMethod( + client.getHttpClient(), addUrl, Utils.JSONCONSUMER, httpPost); + assertNotNull( + "Expected non-null response from POST /cluster/package/testpkg/versions", addResponse); + + // Verify the package was added via GET /cluster/package + String listUrl = baseUrlV2 + "/cluster/package"; + Object listResponse = + HttpClientUtil.executeGET(client.getHttpClient(), listUrl, Utils.JSONCONSUMER); + assertNotNull(listResponse); + + // Verify the package appears in the list + @SuppressWarnings("unchecked") + List versions = + (List) Utils.getObjectByPath(listResponse, true, "result/packages/testpkg"); + assertNotNull("Expected testpkg to be present in packages", versions); + assertFalse("Expected at least one version", versions.isEmpty()); + + // Verify GET /cluster/package/{name} returns only this package + String getByNameUrl = baseUrlV2 + "/cluster/package/testpkg"; + Object getByNameResponse = + HttpClientUtil.executeGET(client.getHttpClient(), getByNameUrl, Utils.JSONCONSUMER); + assertNotNull(getByNameResponse); + @SuppressWarnings("unchecked") + List versionsFromGet = + (List) Utils.getObjectByPath(getByNameResponse, true, "result/packages/testpkg"); + assertNotNull("Expected testpkg in GET by name response", versionsFromGet); + + // Delete the package version via DELETE /cluster/package/{name}/versions/{version} + String deleteUrl = baseUrlV2 + "/cluster/package/testpkg/versions/1.0"; + HttpDelete httpDelete = new HttpDelete(deleteUrl); + Object deleteResponse = + HttpClientUtil.executeHttpMethod( + client.getHttpClient(), deleteUrl, Utils.JSONCONSUMER, httpDelete); + assertNotNull("Expected non-null response from DELETE", deleteResponse); + } + } + + @Test + public void testAddPackageVersionValidatesFiles() throws Exception { + URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); + String baseUrlV2 = cluster.getJettySolrRunner(0).getBaseURLV2().toString(); + + try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { + // Try to add a package version with a non-existent file + String addUrl = baseUrlV2 + "/cluster/package/testpkg2/versions"; + HttpPost httpPost = new HttpPost(addUrl); + httpPost.setHeader("Content-Type", "application/json"); + httpPost.setEntity( + new StringEntity("{\"version\":\"1.0\",\"files\":[\"/nonexistent/file.jar\"]}")); + + org.apache.http.HttpResponse httpResponse = client.getHttpClient().execute(httpPost); + int statusCode = httpResponse.getStatusLine().getStatusCode(); + assertEquals("Expected 400 BAD_REQUEST when specifying non-existent file", 400, statusCode); + } + } + + @Test + public void testRefreshNonExistentPackage() throws Exception { + URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); + String baseUrlV2 = cluster.getJettySolrRunner(0).getBaseURLV2().toString(); + + try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { + // Try to refresh a non-existent package + String refreshUrl = baseUrlV2 + "/cluster/package/nonexistentpkg/refresh"; + HttpPost httpPost = new HttpPost(refreshUrl); + httpPost.setHeader("Content-Type", "application/json"); + + org.apache.http.HttpResponse httpResponse = client.getHttpClient().execute(httpPost); + int statusCode = httpResponse.getStatusLine().getStatusCode(); + assertEquals( + "Expected 400 BAD_REQUEST when refreshing non-existent package", 400, statusCode); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 0804bbee9a1d..685c133c3e56 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -656,7 +656,8 @@ public void testAPI() throws Exception { // we are expecting an error expectError(deleteReq, cluster.getSolrClient(), errPath, "No such version:"); - new V2Request.Builder("/cluster/package/test_pkg/versions/0.12") // correct version. Should succeed + new V2Request.Builder( + "/cluster/package/test_pkg/versions/0.12") // correct version. Should succeed .forceV2(true) .withMethod(SolrRequest.METHOD.DELETE) .build() diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc index 323aa9a3bfe3..4814ebb929a4 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc @@ -178,18 +178,19 @@ For example: == API Endpoints -* `GET /api/cluster/package` Get the list of packages -* `POST /api/cluster/package` edit packages -** `add` command: add a version of a package -** `delete` command: delete a version of a package +* `GET /api/cluster/package` Get the list of all packages +* `GET /api/cluster/package/{name}` Get the details of a specific package +* `POST /api/cluster/package/{name}/versions` Add a version of a package +* `DELETE /api/cluster/package/{name}/versions/{version}` Delete a specific version of a package +* `POST /api/cluster/package/{name}/refresh` Refresh a package on all nodes in the cluster === How to Upgrade? -Use the `add` command to add a version that is higher than the current version. +Use the add version endpoint to add a version that is higher than the current version. === How to Downgrade? -Use the `delete` command to delete the highest version and choose the next highest version. +Use the delete version endpoint to delete the highest version and choose the next highest version. === Using Multiple Versions in Parallel @@ -244,11 +245,9 @@ The plugins loaded from packages cannot depend on core level classes. + [source,bash] ---- -curl http://localhost:8983/api/cluster/package -H 'Content-type:application/json' -d ' -{"add": { - "package" : "mypkg", - "version":"1.0", - "files" :["/mypkg/1.0/myplugins.jar"]}}' +curl -X POST http://localhost:8983/api/cluster/package/mypkg/versions \ + -H 'Content-type:application/json' \ + -d '{"version":"1.0","files":["/mypkg/1.0/myplugins.jar"]}' ---- . Verify the created package: @@ -350,11 +349,9 @@ curl http://localhost:8983/api/cluster/filestore/metadata/mypkg/2.0?omitHeader=t + [source,bash] ---- -curl http://localhost:8983/api/cluster/package -H 'Content-type:application/json' -d ' -{"add": { - "package" : "mypkg", - "version":"2.0", - "files" :["/mypkg/2.0/myplugins.jar"]}}' +curl -X POST http://localhost:8983/api/cluster/package/mypkg/versions \ + -H 'Content-type:application/json' \ + -d '{"version":"2.0","files":["/mypkg/2.0/myplugins.jar"]}' ---- . Verify the plugin to see if the correct version of the package is being used: From cc2641206f6795e9ffc33a4c152eb928609a5b8f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Mar 2026 14:32:38 +0000 Subject: [PATCH 04/21] Fix PackageAPIJaxRsTest to use generated request classes and check response.error field Co-authored-by: epugh <22395+epugh@users.noreply.github.com> --- .../apache/solr/pkg/PackageAPIJaxRsTest.java | 152 ++++++++---------- 1 file changed, 68 insertions(+), 84 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java index ecfaf7b965b2..8e3c1f4b72ca 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java @@ -18,15 +18,11 @@ import static org.apache.solr.filestore.TestDistribFileStore.uploadKey; -import java.net.URL; import java.util.List; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; -import org.apache.solr.client.solrj.apache.HttpClientUtil; +import org.apache.solr.client.api.model.PackagesResponse; import org.apache.solr.client.solrj.apache.HttpSolrClient; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.util.Utils; import org.apache.solr.filestore.ClusterFileStore; import org.junit.BeforeClass; import org.junit.Test; @@ -49,116 +45,104 @@ public static void setupCluster() throws Exception { } @Test - public void testListPackagesReturnsEmptyResult() throws Exception { - URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); - String packageUrl = - cluster.getJettySolrRunner(0).getBaseURLV2().toString() + "/cluster/package"; - - try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { - Object response = - HttpClientUtil.executeGET(client.getHttpClient(), packageUrl, Utils.JSONCONSUMER); + public void testListPackagesReturnsResult() throws Exception { + try (HttpSolrClient client = + new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { + PackagesResponse response = new PackageApi.ListPackages().process(client); assertNotNull("Expected non-null response from GET /cluster/package", response); - // The response should have a 'result' field with 'packages' and 'znodeVersion' - Object result = Utils.getObjectByPath(response, true, "result"); - assertNotNull("Expected 'result' field in GET /cluster/package response", result); + assertNotNull("Expected 'result' field in GET /cluster/package response", response.result); } } @Test - public void testAddDeletePackageVersion() throws Exception { - URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); - String baseUrlV2 = cluster.getJettySolrRunner(0).getBaseURLV2().toString(); - String FILE1 = "/testpkg/runtimelibs.jar"; + public void testAddAndDeletePackageVersion() throws Exception { + String FILE1 = "/jaxrstestpkg/runtimelibs.jar"; // Upload a key and a signed jar file to the filestore byte[] derFile = org.apache.solr.filestore.TestDistribFileStore.readFile("cryptokeys/pub_key512.der"); uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); - org.apache.solr.pkg.TestPackages.postFileAndWait( + TestPackages.postFileAndWait( cluster, "runtimecode/runtimelibs.jar.bin", FILE1, "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { + try (HttpSolrClient client = + new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { // Add a package version via POST /cluster/package/{name}/versions - String addUrl = baseUrlV2 + "/cluster/package/testpkg/versions"; - HttpPost httpPost = new HttpPost(addUrl); - httpPost.setHeader("Content-Type", "application/json"); - httpPost.setEntity(new StringEntity("{\"version\":\"1.0\",\"files\":[\"" + FILE1 + "\"]}")); - Object addResponse = - HttpClientUtil.executeHttpMethod( - client.getHttpClient(), addUrl, Utils.JSONCONSUMER, httpPost); - assertNotNull( - "Expected non-null response from POST /cluster/package/testpkg/versions", addResponse); + PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("jaxrstestpkg"); + addRequest.setVersion("1.0"); + addRequest.setFiles(List.of(FILE1)); + addRequest.process(client); // Verify the package was added via GET /cluster/package - String listUrl = baseUrlV2 + "/cluster/package"; - Object listResponse = - HttpClientUtil.executeGET(client.getHttpClient(), listUrl, Utils.JSONCONSUMER); - assertNotNull(listResponse); - - // Verify the package appears in the list - @SuppressWarnings("unchecked") - List versions = - (List) Utils.getObjectByPath(listResponse, true, "result/packages/testpkg"); - assertNotNull("Expected testpkg to be present in packages", versions); - assertFalse("Expected at least one version", versions.isEmpty()); + PackagesResponse listResponse = new PackageApi.ListPackages().process(client); + assertNotNull("Expected non-null list response", listResponse); + assertNotNull("Expected non-null result", listResponse.result); + assertNotNull( + "Expected jaxrstestpkg in packages", listResponse.result.packages.get("jaxrstestpkg")); + assertFalse( + "Expected at least one version", + listResponse.result.packages.get("jaxrstestpkg").isEmpty()); // Verify GET /cluster/package/{name} returns only this package - String getByNameUrl = baseUrlV2 + "/cluster/package/testpkg"; - Object getByNameResponse = - HttpClientUtil.executeGET(client.getHttpClient(), getByNameUrl, Utils.JSONCONSUMER); - assertNotNull(getByNameResponse); - @SuppressWarnings("unchecked") - List versionsFromGet = - (List) Utils.getObjectByPath(getByNameResponse, true, "result/packages/testpkg"); - assertNotNull("Expected testpkg in GET by name response", versionsFromGet); + PackagesResponse getByNameResponse = + new PackageApi.GetPackage("jaxrstestpkg").process(client); + assertNotNull("Expected non-null get-by-name response", getByNameResponse); + assertNotNull("Expected non-null result from get-by-name", getByNameResponse.result); + assertNotNull( + "Expected jaxrstestpkg in get-by-name response", + getByNameResponse.result.packages.get("jaxrstestpkg")); // Delete the package version via DELETE /cluster/package/{name}/versions/{version} - String deleteUrl = baseUrlV2 + "/cluster/package/testpkg/versions/1.0"; - HttpDelete httpDelete = new HttpDelete(deleteUrl); - Object deleteResponse = - HttpClientUtil.executeHttpMethod( - client.getHttpClient(), deleteUrl, Utils.JSONCONSUMER, httpDelete); - assertNotNull("Expected non-null response from DELETE", deleteResponse); + new PackageApi.DeletePackageVersion("jaxrstestpkg", "1.0").process(client); + + // Verify it's deleted + PackagesResponse listAfterDelete = new PackageApi.ListPackages().process(client); + assertNotNull("Expected non-null list response after delete", listAfterDelete); + assertNotNull("Expected non-null result after delete", listAfterDelete.result); + // After deleting the only version, the package entry should be empty or absent + List versionsAfterDelete = listAfterDelete.result.packages.get("jaxrstestpkg"); + assertTrue( + "Expected no versions after delete", + versionsAfterDelete == null || versionsAfterDelete.isEmpty()); } } @Test public void testAddPackageVersionValidatesFiles() throws Exception { - URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); - String baseUrlV2 = cluster.getJettySolrRunner(0).getBaseURLV2().toString(); - - try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { - // Try to add a package version with a non-existent file - String addUrl = baseUrlV2 + "/cluster/package/testpkg2/versions"; - HttpPost httpPost = new HttpPost(addUrl); - httpPost.setHeader("Content-Type", "application/json"); - httpPost.setEntity( - new StringEntity("{\"version\":\"1.0\",\"files\":[\"/nonexistent/file.jar\"]}")); - - org.apache.http.HttpResponse httpResponse = client.getHttpClient().execute(httpPost); - int statusCode = httpResponse.getStatusLine().getStatusCode(); - assertEquals("Expected 400 BAD_REQUEST when specifying non-existent file", 400, statusCode); + try (HttpSolrClient client = + new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { + // Try to add a package version with a non-existent file. + // Note: JacksonDataBindResponseParser doesn't expose errors as RemoteSolrException; + // instead, the error is in the response body's 'error' field. + PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("testpkg_invalid"); + addRequest.setVersion("1.0"); + addRequest.setFiles(List.of("/nonexistent/file.jar")); + + var response = addRequest.process(client); + assertNotNull("Expected error in response for non-existent file", response.error); + assertEquals("Expected 400 for non-existent file", 400, (int) response.error.code); + assertTrue( + "Expected error message to mention the file", + response.error.msg.contains("No such file")); } } @Test public void testRefreshNonExistentPackage() throws Exception { - URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); - String baseUrlV2 = cluster.getJettySolrRunner(0).getBaseURLV2().toString(); - - try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { - // Try to refresh a non-existent package - String refreshUrl = baseUrlV2 + "/cluster/package/nonexistentpkg/refresh"; - HttpPost httpPost = new HttpPost(refreshUrl); - httpPost.setHeader("Content-Type", "application/json"); - - org.apache.http.HttpResponse httpResponse = client.getHttpClient().execute(httpPost); - int statusCode = httpResponse.getStatusLine().getStatusCode(); - assertEquals( - "Expected 400 BAD_REQUEST when refreshing non-existent package", 400, statusCode); + try (HttpSolrClient client = + new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { + // Try to refresh a non-existent package. + // Note: JacksonDataBindResponseParser doesn't expose errors as RemoteSolrException; + // instead, the error is in the response body's 'error' field. + var response = new PackageApi.RefreshPackage("nonexistentpkg_jaxrs").process(client); + assertNotNull("Expected error in response for non-existent package", response.error); + assertEquals("Expected 400 for non-existent package", 400, (int) response.error.code); + assertTrue( + "Expected error message to mention the package", + response.error.msg.contains("No such package")); } } } From 5040f5b47b0b1c20e27ad422d56178d8b704ec31 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Mar 2026 14:33:45 +0000 Subject: [PATCH 05/21] Extract magic numbers as named constants in PackageAPIJaxRs Co-authored-by: epugh <22395+epugh@users.noreply.github.com> --- .../core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java index 9d72b1cff543..92147673c5e6 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java @@ -58,6 +58,9 @@ public class PackageAPIJaxRs extends JerseyResource implements PackageApis { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final int SYNC_MAX_RETRIES = 10; + private static final long SYNC_SLEEP_MS = 10L; + private final CoreContainer coreContainer; private final SolrQueryRequest solrQueryRequest; private final SolrQueryResponse solrQueryResponse; @@ -266,7 +269,7 @@ public SolrJerseyResponse refreshPackage(String packageName) { private void syncToVersion(PackageAPI packageAPI, int expectedVersion) { int origVersion = packageAPI.pkgs.znodeVersion; - for (int i = 0; i < 10; i++) { + for (int i = 0; i < SYNC_MAX_RETRIES; i++) { if (log.isDebugEnabled()) { log.debug( "my version is {} , and expected version {}", @@ -280,7 +283,7 @@ private void syncToVersion(PackageAPI packageAPI, int expectedVersion) { return; } try { - Thread.sleep(10); + Thread.sleep(SYNC_SLEEP_MS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } From 83c2f5c36c7ff96db4bbebd7386af5ecd2b9b5c9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Mar 2026 16:13:55 +0000 Subject: [PATCH 06/21] Rename PackageAPIJaxRs to PackageAPI and old PackageAPI to PackageStore Co-authored-by: epugh <22395+epugh@users.noreply.github.com> --- .../solr/api/ContainerPluginsRegistry.java | 2 +- .../org/apache/solr/core/CoreContainer.java | 4 +- .../solr/filestore/ClusterFileStore.java | 12 +- .../solr/handler/SolrConfigHandler.java | 4 +- .../solr/handler/component/SearchHandler.java | 4 +- .../packagemanager/RepositoryManager.java | 1 - .../java/org/apache/solr/pkg/PackageAPI.java | 431 +++++++++++------- .../org/apache/solr/pkg/PackageAPIJaxRs.java | 352 -------------- .../org/apache/solr/pkg/PackageListeners.java | 2 +- .../solr/pkg/PackageListeningClassLoader.java | 12 +- .../apache/solr/pkg/PackagePluginHolder.java | 2 +- .../org/apache/solr/pkg/PackageStore.java | 241 ++++++++++ .../apache/solr/pkg/SolrPackageLoader.java | 40 +- .../solr/handler/TestContainerPlugin.java | 4 +- ...eAPIJaxRsTest.java => PackageAPITest.java} | 24 +- 15 files changed, 568 insertions(+), 567 deletions(-) delete mode 100644 solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java create mode 100644 solr/core/src/java/org/apache/solr/pkg/PackageStore.java rename solr/core/src/test/org/apache/solr/pkg/{PackageAPIJaxRsTest.java => PackageAPITest.java} (89%) diff --git a/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java b/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java index 20cd2440bbe3..f58cb95fbcb5 100644 --- a/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java +++ b/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java @@ -335,7 +335,7 @@ public ApiInfo(PluginMetaHolder infoHolder, List errs) { coreContainer.getPackageLoader().getPackageVersion(pkg, info.version); if (ver.isEmpty()) { // may be we are a bit early. Do a refresh and try again - coreContainer.getPackageLoader().getPackageAPI().refreshPackages(null); + coreContainer.getPackageLoader().getPackageStore().refreshPackages(null); ver = coreContainer.getPackageLoader().getPackageVersion(pkg, info.version); } if (ver.isEmpty()) { diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 04d7450aa645..66daf22de020 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -136,7 +136,7 @@ import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.metrics.otel.OtelUnit; -import org.apache.solr.pkg.PackageAPIJaxRs; +import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequestBase; @@ -842,7 +842,7 @@ private void loadInternal() { registerV2ApiIfEnabled(ClusterFileStore.class); packageLoader = new SolrPackageLoader(this); - registerV2ApiIfEnabled(PackageAPIJaxRs.class); + registerV2ApiIfEnabled(PackageAPI.class); registerV2ApiIfEnabled(ZookeeperRead.class); } diff --git a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java index c3b76dca4bba..bdda47d55f7d 100644 --- a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java +++ b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java @@ -47,7 +47,7 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; import org.apache.solr.jersey.PermissionName; -import org.apache.solr.pkg.PackageAPI; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.PermissionNameProvider; @@ -86,8 +86,8 @@ public ClusterFileStore( public UploadToFileStoreResponse uploadFile( String filePath, List sig, InputStream requestBody) { final var response = instantiateJerseyResponse(UploadToFileStoreResponse.class); - if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { - throw new RuntimeException(PackageAPI.ERR_MSG); + if (!coreContainer.getPackageLoader().getPackageStore().isEnabled()) { + throw new RuntimeException(PackageStore.ERR_MSG); } try { coreContainer @@ -302,12 +302,12 @@ private void doDelete(String filePath, Boolean localDelete) { @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM) public SolrJerseyResponse deleteFile(String filePath, Boolean localDelete) { final var response = instantiateJerseyResponse(SolrJerseyResponse.class); - if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { - throw new RuntimeException(PackageAPI.ERR_MSG); + if (!coreContainer.getPackageLoader().getPackageStore().isEnabled()) { + throw new RuntimeException(PackageStore.ERR_MSG); } validateName(filePath, true); - if (coreContainer.getPackageLoader().getPackageAPI().isJarInuse(filePath)) { + if (coreContainer.getPackageLoader().getPackageStore().isJarInuse(filePath)) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "jar in use, can't delete"); } doDelete(filePath, localDelete); diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java index cba47bad98d2..e7a7f9f98f4d 100644 --- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java @@ -88,8 +88,8 @@ import org.apache.solr.handler.admin.api.GetConfigAPI; import org.apache.solr.handler.admin.api.ModifyConfigComponentAPI; import org.apache.solr.handler.admin.api.ModifyParamSetAPI; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.response.SolrQueryResponse; @@ -303,7 +303,7 @@ private void handleGET() { List listeners = req.getCore().getPackageListeners().getListeners(); for (PackageListeners.Listener listener : listeners) { - Map infos = listener.packageDetails(); + Map infos = listener.packageDetails(); if (infos == null || infos.isEmpty()) continue; infos.forEach( (s, mapWriter) -> { diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java index d3bb6216b8ae..54b434eca793 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java @@ -62,8 +62,8 @@ import org.apache.solr.core.SolrCore; import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.metrics.SolrMetricsContext; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -202,7 +202,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return Collections.emptyMap(); } diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java index 013d3639e55b..5ae49bd02a39 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java @@ -58,7 +58,6 @@ import org.apache.solr.filestore.ClusterFileStore; import org.apache.solr.packagemanager.SolrPackage.Artifact; import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index d2394aefe917..0e0318bd0505 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -14,226 +14,339 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.solr.pkg; import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; +import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM; +import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM; -import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.inject.Inject; import java.io.IOException; -import java.io.StringWriter; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; +import org.apache.solr.api.JerseyResource; +import org.apache.solr.client.api.endpoint.PackageApis; import org.apache.solr.client.api.model.AddPackageVersionRequestBody; +import org.apache.solr.client.api.model.PackagesResponse; +import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.response.JavaBinResponseParser; import org.apache.solr.common.SolrException; -import org.apache.solr.common.annotation.JsonProperty; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZooKeeperException; -import org.apache.solr.common.util.EnvUtils; -import org.apache.solr.common.util.ReflectMapWriter; +import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; -import org.apache.solr.util.SolrJacksonAnnotationInspector; +import org.apache.solr.filestore.FileStoreUtils; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** This implements the public end points (/api/cluster/package) of package API. */ -public class PackageAPI { - public final boolean enablePackages = EnvUtils.getPropertyAsBool("solr.packages.enabled", false); +/** + * JAX-RS implementation of the package management API ({@code /api/cluster/package}). + * + * @see PackageApis + */ +public class PackageAPI extends JerseyResource implements PackageApis { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final String ERR_MSG = - "Package loading is not enabled , Start your nodes with -Dsolr.packages.enabled=true"; + private static final int SYNC_MAX_RETRIES = 10; + private static final long SYNC_SLEEP_MS = 10L; - final CoreContainer coreContainer; - final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); - final SolrPackageLoader packageLoader; - Packages pkgs; + private final CoreContainer coreContainer; + private final SolrQueryRequest solrQueryRequest; + private final SolrQueryResponse solrQueryResponse; - public PackageAPI(CoreContainer coreContainer, SolrPackageLoader loader) { + @Inject + public PackageAPI( + CoreContainer coreContainer, + SolrQueryRequest solrQueryRequest, + SolrQueryResponse solrQueryResponse) { this.coreContainer = coreContainer; - this.packageLoader = loader; - pkgs = new Packages(); - SolrZkClient zkClient = coreContainer.getZkController().getZkClient(); - try { - pkgs = readPkgsFromZk(null, null); - } catch (KeeperException | InterruptedException e) { - pkgs = new Packages(); - // ignore + this.solrQueryRequest = solrQueryRequest; + this.solrQueryResponse = solrQueryResponse; + } + + @Override + @PermissionName(PACKAGE_READ_PERM) + public PackagesResponse listPackages(String refreshPackage, Integer expectedVersion) { + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + + if (refreshPackage != null) { + packageStore.packageLoader.notifyListeners(refreshPackage); + return instantiateJerseyResponse(PackagesResponse.class); } - try { - registerListener(zkClient); - } catch (KeeperException | InterruptedException e) { - SolrZkClient.checkInterrupted(e); + + if (expectedVersion != null) { + syncToVersion(packageStore, expectedVersion); } - } - private void registerListener(SolrZkClient zkClient) - throws KeeperException, InterruptedException { - zkClient.exists( - SOLR_PKGS_PATH, - new Watcher() { - - @Override - public void process(WatchedEvent event) { - // session events are not change events, and do not remove the watcher - if (Event.EventType.None.equals(event.getType())) { - return; - } - synchronized (this) { - log.debug("Updating [{}] ... ", SOLR_PKGS_PATH); - // remake watch - final Watcher thisWatch = this; - refreshPackages(thisWatch); - } - } - }); + final var response = instantiateJerseyResponse(PackagesResponse.class); + response.result = toPackageData(packageStore.pkgs); + return response; } - public void refreshPackages(Watcher watcher) { - final Stat stat = new Stat(); - try { - final byte[] data = - coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, watcher, stat); - pkgs = readPkgsFromZk(data, stat); - packageLoader.refreshPackageConf(); - } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) { - log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: ", e); - } catch (KeeperException e) { - log.error("A ZK error has occurred", e); - throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); - } catch (InterruptedException e) { - // Restore the interrupted status - Thread.currentThread().interrupt(); - log.warn("Interrupted", e); + @Override + @PermissionName(PACKAGE_READ_PERM) + public PackagesResponse getPackage(String packageName) { + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + final var response = instantiateJerseyResponse(PackagesResponse.class); + response.result = toPackageData(packageStore.pkgs); + // Filter to only the requested package + if (response.result != null && response.result.packages != null) { + final var pkgVersions = response.result.packages.get(packageName); + response.result.packages = Collections.singletonMap(packageName, pkgVersions); } + return response; } - Packages readPkgsFromZk(byte[] data, Stat stat) throws KeeperException, InterruptedException { + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse addPackageVersion( + String packageName, AddPackageVersionRequestBody requestBody) { + final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); - if (data == null || stat == null) { - stat = new Stat(); - data = coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, null, stat); + if (!packageStore.isEnabled()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, PackageStore.ERR_MSG); } - Packages packages = null; - if (data == null || data.length == 0) { - packages = new Packages(); - } else { - try { - packages = mapper.readValue(data, Packages.class); - packages.znodeVersion = stat.getVersion(); - } catch (IOException e) { - // invalid data in packages - // TODO handle properly; - return new Packages(); - } + if (requestBody == null || requestBody.files == null || requestBody.files.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No files specified"); } - return packages; - } - public static class Packages implements ReflectMapWriter { - @JsonProperty public int znodeVersion = -1; + final List errors = new ArrayList<>(); + FileStoreUtils.validateFiles( + coreContainer.getFileStore(), requestBody.files, true, errors::add); + if (!errors.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, String.join("; ", errors)); + } - @JsonProperty public Map> packages = new LinkedHashMap<>(); + final PackageStore.Packages[] finalState = new PackageStore.Packages[1]; + try { + coreContainer + .getZkController() + .getZkClient() + .atomicUpdate( + SOLR_PKGS_PATH, + (stat, bytes) -> { + PackageStore.Packages packages; + try { + packages = + bytes == null + ? new PackageStore.Packages() + : packageStore.mapper.readValue(bytes, PackageStore.Packages.class); + packages = packages.copy(); + } catch (IOException e) { + log.error("Error deserializing packages.json", e); + packages = new PackageStore.Packages(); + } + List list = + packages.packages.computeIfAbsent(packageName, o -> new ArrayList<>()); + for (PackageStore.PkgVersion pkgVersion : list) { + if (Objects.equals(pkgVersion.version, requestBody.version)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Version '" + requestBody.version + "' exists already"); + } + } + list.add(new PackageStore.PkgVersion(packageName, requestBody)); + packages.znodeVersion = stat.getVersion() + 1; + finalState[0] = packages; + return Utils.toJSON(packages); + }); + } catch (KeeperException | InterruptedException e) { + finalState[0] = null; + packageStore.handleZkErr(e); + } - public Packages copy() { - Packages p = new Packages(); - p.znodeVersion = this.znodeVersion; - p.packages = new LinkedHashMap<>(); - packages.forEach((s, versions) -> p.packages.put(s, new ArrayList<>(versions))); - return p; + if (finalState[0] != null) { + packageStore.pkgs = finalState[0]; + notifyAllNodesToSync(packageStore.pkgs.znodeVersion); + coreContainer.getPackageLoader().refreshPackageConf(); } - } - public static class PkgVersion implements ReflectMapWriter { + return response; + } - @JsonProperty("package") - public String pkg; + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse deletePackageVersion(String packageName, String version) { + final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); - @JsonProperty public String version; + if (!packageStore.isEnabled()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, PackageStore.ERR_MSG); + } - @JsonProperty public List files; + try { + coreContainer + .getZkController() + .getZkClient() + .atomicUpdate( + SOLR_PKGS_PATH, + (stat, bytes) -> { + PackageStore.Packages packages; + try { + packages = packageStore.mapper.readValue(bytes, PackageStore.Packages.class); + packages = packages.copy(); + } catch (IOException e) { + packages = new PackageStore.Packages(); + } - @JsonProperty public String manifest; + List versions = packages.packages.get(packageName); + if (versions == null || versions.isEmpty()) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No such package: " + packageName); + } + int idxToRemove = -1; + for (int i = 0; i < versions.size(); i++) { + if (Objects.equals(versions.get(i).version, version)) { + idxToRemove = i; + break; + } + } + if (idxToRemove == -1) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No such version: " + version); + } + versions.remove(idxToRemove); + packages.znodeVersion = stat.getVersion() + 1; + return Utils.toJSON(packages); + }); + } catch (KeeperException | InterruptedException e) { + packageStore.handleZkErr(e); + } - @JsonProperty public String manifestSHA512; + return response; + } - public PkgVersion() {} + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse refreshPackage(String packageName) { + final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); - public PkgVersion(String packageName, AddPackageVersionRequestBody addVersion) { - this.pkg = packageName; - this.version = addVersion.version; - this.files = addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); - this.manifest = addVersion.manifest; - this.manifestSHA512 = addVersion.manifestSHA512; + SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(packageName); + if (pkg == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No such package: " + packageName); } + // first refresh on the current node + packageStore.packageLoader.notifyListeners(packageName); - @Override - public boolean equals(Object obj) { - if (obj instanceof PkgVersion that) { - return Objects.equals(this.version, that.version) && Objects.equals(this.files, that.files); - } - return false; - } + final var solrParams = new ModifiableSolrParams(); + solrParams.add("omitHeader", "true"); + solrParams.add("refreshPackage", packageName); - @Override - public int hashCode() { - return Objects.hash(version); - } + final var request = + new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); + request.setResponseParser(new JavaBinResponseParser()); - @Override - public String toString() { + for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { + final var baseUrl = + coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); try { - return Utils.writeJson(this, new StringWriter(), false).toString(); - } catch (IOException e) { - throw new RuntimeException(e); + var solrClient = coreContainer.getDefaultHttpSolrClient(); + solrClient.requestWithBaseUrl(baseUrl, request::process); + } catch (SolrServerException | IOException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Failed to refresh package on node: " + liveNode, + e); } } - public PkgVersion copy() { - PkgVersion result = new PkgVersion(); - result.pkg = this.pkg; - result.version = this.version; - result.files = this.files; - result.manifest = this.manifest; - result.manifestSHA512 = this.manifestSHA512; - return result; - } + return response; } - public boolean isEnabled() { - return enablePackages; + private void syncToVersion(PackageStore packageStore, int expectedVersion) { + int origVersion = packageStore.pkgs.znodeVersion; + for (int i = 0; i < SYNC_MAX_RETRIES; i++) { + if (log.isDebugEnabled()) { + log.debug( + "my version is {} , and expected version {}", + packageStore.pkgs.znodeVersion, + expectedVersion); + } + if (packageStore.pkgs.znodeVersion >= expectedVersion) { + if (origVersion < packageStore.pkgs.znodeVersion) { + coreContainer.getPackageLoader().refreshPackageConf(); + } + return; + } + try { + Thread.sleep(SYNC_SLEEP_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + try { + packageStore.pkgs = packageStore.readPkgsFromZk(null, null); + } catch (KeeperException | InterruptedException e) { + packageStore.handleZkErr(e); + } + } } - public void handleZkErr(Exception e) { - log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e)); - } + private void notifyAllNodesToSync(int expectedVersion) { + final var solrParams = new ModifiableSolrParams(); + solrParams.add("omitHeader", "true"); + solrParams.add("expectedVersion", String.valueOf(expectedVersion)); - public boolean isJarInuse(String path) { - Packages pkg = null; - try { - pkg = readPkgsFromZk(null, null); - } catch (KeeperException.NoNodeException nne) { - return false; - } catch (InterruptedException | KeeperException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } - for (List vers : pkg.packages.values()) { - for (PkgVersion ver : vers) { - if (ver.files.contains(path)) { - return true; - } + final var request = + new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); + request.setResponseParser(new JavaBinResponseParser()); + + for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { + var baseUrl = coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); + try { + var solrClient = coreContainer.getDefaultHttpSolrClient(); + solrClient.requestWithBaseUrl(baseUrl, request::process); + } catch (SolrServerException | IOException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Failed to notify node: " + + liveNode + + " to sync expected package version: " + + expectedVersion, + e); } } - return false; + } + + private static PackagesResponse.PackageData toPackageData(PackageStore.Packages packages) { + if (packages == null) { + return null; + } + final var data = new PackagesResponse.PackageData(); + data.znodeVersion = packages.znodeVersion; + data.packages = + packages.packages.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + e -> + e.getValue().stream() + .map(PackageAPI::toPkgVersionResponse) + .collect(Collectors.toList()))); + return data; + } + + private static PackagesResponse.PackageVersion toPkgVersionResponse( + PackageStore.PkgVersion pkgVersion) { + final var v = new PackagesResponse.PackageVersion(); + v.pkg = pkgVersion.pkg; + v.version = pkgVersion.version; + v.files = pkgVersion.files; + v.manifest = pkgVersion.manifest; + v.manifestSHA512 = pkgVersion.manifestSHA512; + return v; } } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java deleted file mode 100644 index 92147673c5e6..000000000000 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java +++ /dev/null @@ -1,352 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.pkg; - -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; -import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM; -import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM; - -import jakarta.inject.Inject; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; -import org.apache.solr.api.JerseyResource; -import org.apache.solr.client.api.endpoint.PackageApis; -import org.apache.solr.client.api.model.AddPackageVersionRequestBody; -import org.apache.solr.client.api.model.PackagesResponse; -import org.apache.solr.client.api.model.SolrJerseyResponse; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.response.JavaBinResponseParser; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.filestore.FileStoreUtils; -import org.apache.solr.jersey.PermissionName; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; -import org.apache.zookeeper.KeeperException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * JAX-RS implementation of the package management API ({@code /api/cluster/package}). - * - * @see PackageApis - */ -public class PackageAPIJaxRs extends JerseyResource implements PackageApis { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final int SYNC_MAX_RETRIES = 10; - private static final long SYNC_SLEEP_MS = 10L; - - private final CoreContainer coreContainer; - private final SolrQueryRequest solrQueryRequest; - private final SolrQueryResponse solrQueryResponse; - - @Inject - public PackageAPIJaxRs( - CoreContainer coreContainer, - SolrQueryRequest solrQueryRequest, - SolrQueryResponse solrQueryResponse) { - this.coreContainer = coreContainer; - this.solrQueryRequest = solrQueryRequest; - this.solrQueryResponse = solrQueryResponse; - } - - @Override - @PermissionName(PACKAGE_READ_PERM) - public PackagesResponse listPackages(String refreshPackage, Integer expectedVersion) { - PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); - - if (refreshPackage != null) { - packageAPI.packageLoader.notifyListeners(refreshPackage); - return instantiateJerseyResponse(PackagesResponse.class); - } - - if (expectedVersion != null) { - syncToVersion(packageAPI, expectedVersion); - } - - final var response = instantiateJerseyResponse(PackagesResponse.class); - response.result = toPackageData(packageAPI.pkgs); - return response; - } - - @Override - @PermissionName(PACKAGE_READ_PERM) - public PackagesResponse getPackage(String packageName) { - PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); - final var response = instantiateJerseyResponse(PackagesResponse.class); - response.result = toPackageData(packageAPI.pkgs); - // Filter to only the requested package - if (response.result != null && response.result.packages != null) { - final var pkgVersions = response.result.packages.get(packageName); - response.result.packages = Collections.singletonMap(packageName, pkgVersions); - } - return response; - } - - @Override - @PermissionName(PACKAGE_EDIT_PERM) - public SolrJerseyResponse addPackageVersion( - String packageName, AddPackageVersionRequestBody requestBody) { - final var response = instantiateJerseyResponse(SolrJerseyResponse.class); - PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); - - if (!packageAPI.isEnabled()) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, PackageAPI.ERR_MSG); - } - if (requestBody == null || requestBody.files == null || requestBody.files.isEmpty()) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No files specified"); - } - - final List errors = new ArrayList<>(); - FileStoreUtils.validateFiles( - coreContainer.getFileStore(), requestBody.files, true, errors::add); - if (!errors.isEmpty()) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, String.join("; ", errors)); - } - - final PackageAPI.Packages[] finalState = new PackageAPI.Packages[1]; - try { - coreContainer - .getZkController() - .getZkClient() - .atomicUpdate( - SOLR_PKGS_PATH, - (stat, bytes) -> { - PackageAPI.Packages packages; - try { - packages = - bytes == null - ? new PackageAPI.Packages() - : packageAPI.mapper.readValue(bytes, PackageAPI.Packages.class); - packages = packages.copy(); - } catch (IOException e) { - log.error("Error deserializing packages.json", e); - packages = new PackageAPI.Packages(); - } - List list = - packages.packages.computeIfAbsent(packageName, o -> new ArrayList<>()); - for (PackageAPI.PkgVersion pkgVersion : list) { - if (Objects.equals(pkgVersion.version, requestBody.version)) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "Version '" + requestBody.version + "' exists already"); - } - } - list.add(new PackageAPI.PkgVersion(packageName, requestBody)); - packages.znodeVersion = stat.getVersion() + 1; - finalState[0] = packages; - return Utils.toJSON(packages); - }); - } catch (KeeperException | InterruptedException e) { - finalState[0] = null; - packageAPI.handleZkErr(e); - } - - if (finalState[0] != null) { - packageAPI.pkgs = finalState[0]; - notifyAllNodesToSync(packageAPI.pkgs.znodeVersion); - coreContainer.getPackageLoader().refreshPackageConf(); - } - - return response; - } - - @Override - @PermissionName(PACKAGE_EDIT_PERM) - public SolrJerseyResponse deletePackageVersion(String packageName, String version) { - final var response = instantiateJerseyResponse(SolrJerseyResponse.class); - PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); - - if (!packageAPI.isEnabled()) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, PackageAPI.ERR_MSG); - } - - try { - coreContainer - .getZkController() - .getZkClient() - .atomicUpdate( - SOLR_PKGS_PATH, - (stat, bytes) -> { - PackageAPI.Packages packages; - try { - packages = packageAPI.mapper.readValue(bytes, PackageAPI.Packages.class); - packages = packages.copy(); - } catch (IOException e) { - packages = new PackageAPI.Packages(); - } - - List versions = packages.packages.get(packageName); - if (versions == null || versions.isEmpty()) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "No such package: " + packageName); - } - int idxToRemove = -1; - for (int i = 0; i < versions.size(); i++) { - if (Objects.equals(versions.get(i).version, version)) { - idxToRemove = i; - break; - } - } - if (idxToRemove == -1) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "No such version: " + version); - } - versions.remove(idxToRemove); - packages.znodeVersion = stat.getVersion() + 1; - return Utils.toJSON(packages); - }); - } catch (KeeperException | InterruptedException e) { - packageAPI.handleZkErr(e); - } - - return response; - } - - @Override - @PermissionName(PACKAGE_EDIT_PERM) - public SolrJerseyResponse refreshPackage(String packageName) { - final var response = instantiateJerseyResponse(SolrJerseyResponse.class); - PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); - - SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(packageName); - if (pkg == null) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "No such package: " + packageName); - } - // first refresh on the current node - packageAPI.packageLoader.notifyListeners(packageName); - - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("refreshPackage", packageName); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); - - for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - final var baseUrl = - coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); - try { - var solrClient = coreContainer.getDefaultHttpSolrClient(); - solrClient.requestWithBaseUrl(baseUrl, request::process); - } catch (SolrServerException | IOException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to refresh package on node: " + liveNode, - e); - } - } - - return response; - } - - private void syncToVersion(PackageAPI packageAPI, int expectedVersion) { - int origVersion = packageAPI.pkgs.znodeVersion; - for (int i = 0; i < SYNC_MAX_RETRIES; i++) { - if (log.isDebugEnabled()) { - log.debug( - "my version is {} , and expected version {}", - packageAPI.pkgs.znodeVersion, - expectedVersion); - } - if (packageAPI.pkgs.znodeVersion >= expectedVersion) { - if (origVersion < packageAPI.pkgs.znodeVersion) { - coreContainer.getPackageLoader().refreshPackageConf(); - } - return; - } - try { - Thread.sleep(SYNC_SLEEP_MS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - try { - packageAPI.pkgs = packageAPI.readPkgsFromZk(null, null); - } catch (KeeperException | InterruptedException e) { - packageAPI.handleZkErr(e); - } - } - } - - private void notifyAllNodesToSync(int expectedVersion) { - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("expectedVersion", String.valueOf(expectedVersion)); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); - - for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - var baseUrl = coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); - try { - var solrClient = coreContainer.getDefaultHttpSolrClient(); - solrClient.requestWithBaseUrl(baseUrl, request::process); - } catch (SolrServerException | IOException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to notify node: " - + liveNode - + " to sync expected package version: " - + expectedVersion, - e); - } - } - } - - private static PackagesResponse.PackageData toPackageData(PackageAPI.Packages packages) { - if (packages == null) { - return null; - } - final var data = new PackagesResponse.PackageData(); - data.znodeVersion = packages.znodeVersion; - data.packages = - packages.packages.entrySet().stream() - .collect( - Collectors.toMap( - Map.Entry::getKey, - e -> - e.getValue().stream() - .map(PackageAPIJaxRs::toPkgVersionResponse) - .collect(Collectors.toList()))); - return data; - } - - private static PackagesResponse.PackageVersion toPkgVersionResponse( - PackageAPI.PkgVersion pkgVersion) { - final var v = new PackagesResponse.PackageVersion(); - v.pkg = pkgVersion.pkg; - v.version = pkgVersion.version; - v.files = pkgVersion.files; - v.manifest = pkgVersion.manifest; - v.manifestSHA512 = pkgVersion.manifestSHA512; - return v; - } -} diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java index 7535bb2c7fe9..b892ae4a9561 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java @@ -110,7 +110,7 @@ public interface Listener { String packageName(); /** fetch the package versions of class names */ - Map packageDetails(); + Map packageDetails(); /** A callback when the package is updated */ void changed(SolrPackageLoader.SolrPackage pkg, Ctx ctx); diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java index 272de544b6e7..8c9fa0a97889 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java @@ -42,7 +42,7 @@ public class PackageListeningClassLoader implements SolrClassLoader, PackageList private final Function pkgVersionSupplier; /** package name and the versions that we are tracking */ - private Map packageVersions = new ConcurrentHashMap<>(1); + private Map packageVersions = new ConcurrentHashMap<>(1); private Map classNameVsPackageName = new ConcurrentHashMap<>(); private final Runnable reloadAction; @@ -99,7 +99,7 @@ public SolrPackageLoader.SolrPackage.Version findPackageVersion( p.getLatest(pkgVersionSupplier.apply(cName.pkg)); if (registerListener) { classNameVsPackageName.put(cName.original, cName.pkg); - PackageAPI.PkgVersion pkgVersion = theVersion.getPkgVersion(); + PackageStore.PkgVersion pkgVersion = theVersion.getPkgVersion(); if (pkgVersion != null) packageVersions.put(cName.pkg, pkgVersion); } return theVersion; @@ -111,7 +111,7 @@ public SolrPackageLoader.SolrPackage.Version findPackageVersion( @Override public MapWriter getPackageVersion(PluginInfo.ClassName cName) { if (cName.pkg == null) return null; - PackageAPI.PkgVersion p = packageVersions.get(cName.pkg); + PackageStore.PkgVersion p = packageVersions.get(cName.pkg); return p == null ? null : p::writeMap; } @@ -162,15 +162,15 @@ public String packageName() { } @Override - public Map packageDetails() { - Map result = new LinkedHashMap<>(); + public Map packageDetails() { + Map result = new LinkedHashMap<>(); classNameVsPackageName.forEach((k, v) -> result.put(k, packageVersions.get(v))); return result; } @Override public void changed(SolrPackageLoader.SolrPackage pkg, Ctx ctx) { - PackageAPI.PkgVersion currVer = packageVersions.get(pkg.name); + PackageStore.PkgVersion currVer = packageVersions.get(pkg.name); if (currVer == null) { // not watching this return; diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java index b0731d05095f..2ac38f3b7ab3 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java @@ -59,7 +59,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return Collections.singletonMap(info.cName.original, pkgVersion.getPkgVersion()); } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageStore.java b/solr/core/src/java/org/apache/solr/pkg/PackageStore.java new file mode 100644 index 000000000000..77c32899c392 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/pkg/PackageStore.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.pkg; + +import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.io.StringWriter; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.annotation.JsonProperty; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZooKeeperException; +import org.apache.solr.common.util.EnvUtils; +import org.apache.solr.common.util.ReflectMapWriter; +import org.apache.solr.common.util.Utils; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.util.SolrJacksonAnnotationInspector; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Holds package data loaded from ZooKeeper ({@code /solr/packages.json}) and manages ZK watchers. + */ +public class PackageStore { + public final boolean enablePackages = EnvUtils.getPropertyAsBool("solr.packages.enabled", false); + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public static final String ERR_MSG = + "Package loading is not enabled , Start your nodes with -Dsolr.packages.enabled=true"; + + final CoreContainer coreContainer; + final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); + final SolrPackageLoader packageLoader; + Packages pkgs; + + public PackageStore(CoreContainer coreContainer, SolrPackageLoader loader) { + this.coreContainer = coreContainer; + this.packageLoader = loader; + pkgs = new Packages(); + SolrZkClient zkClient = coreContainer.getZkController().getZkClient(); + try { + pkgs = readPkgsFromZk(null, null); + } catch (KeeperException | InterruptedException e) { + pkgs = new Packages(); + // ignore + } + try { + registerListener(zkClient); + } catch (KeeperException | InterruptedException e) { + SolrZkClient.checkInterrupted(e); + } + } + + private void registerListener(SolrZkClient zkClient) + throws KeeperException, InterruptedException { + zkClient.exists( + SOLR_PKGS_PATH, + new Watcher() { + + @Override + public void process(WatchedEvent event) { + // session events are not change events, and do not remove the watcher + if (Event.EventType.None.equals(event.getType())) { + return; + } + synchronized (this) { + log.debug("Updating [{}] ... ", SOLR_PKGS_PATH); + // remake watch + final Watcher thisWatch = this; + refreshPackages(thisWatch); + } + } + }); + } + + public void refreshPackages(Watcher watcher) { + final Stat stat = new Stat(); + try { + final byte[] data = + coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, watcher, stat); + pkgs = readPkgsFromZk(data, stat); + packageLoader.refreshPackageConf(); + } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) { + log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: ", e); + } catch (KeeperException e) { + log.error("A ZK error has occurred", e); + throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); + } catch (InterruptedException e) { + // Restore the interrupted status + Thread.currentThread().interrupt(); + log.warn("Interrupted", e); + } + } + + Packages readPkgsFromZk(byte[] data, Stat stat) throws KeeperException, InterruptedException { + + if (data == null || stat == null) { + stat = new Stat(); + data = coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, null, stat); + } + Packages packages = null; + if (data == null || data.length == 0) { + packages = new Packages(); + } else { + try { + packages = mapper.readValue(data, Packages.class); + packages.znodeVersion = stat.getVersion(); + } catch (IOException e) { + // invalid data in packages + // TODO handle properly; + return new Packages(); + } + } + return packages; + } + + public static class Packages implements ReflectMapWriter { + @JsonProperty public int znodeVersion = -1; + + @JsonProperty public Map> packages = new LinkedHashMap<>(); + + public Packages copy() { + Packages p = new Packages(); + p.znodeVersion = this.znodeVersion; + p.packages = new LinkedHashMap<>(); + packages.forEach((s, versions) -> p.packages.put(s, new ArrayList<>(versions))); + return p; + } + } + + public static class PkgVersion implements ReflectMapWriter { + + @JsonProperty("package") + public String pkg; + + @JsonProperty public String version; + + @JsonProperty public List files; + + @JsonProperty public String manifest; + + @JsonProperty public String manifestSHA512; + + public PkgVersion() {} + + public PkgVersion(String packageName, AddPackageVersionRequestBody addVersion) { + this.pkg = packageName; + this.version = addVersion.version; + this.files = addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); + this.manifest = addVersion.manifest; + this.manifestSHA512 = addVersion.manifestSHA512; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof PkgVersion that) { + return Objects.equals(this.version, that.version) && Objects.equals(this.files, that.files); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(version); + } + + @Override + public String toString() { + try { + return Utils.writeJson(this, new StringWriter(), false).toString(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public PkgVersion copy() { + PkgVersion result = new PkgVersion(); + result.pkg = this.pkg; + result.version = this.version; + result.files = this.files; + result.manifest = this.manifest; + result.manifestSHA512 = this.manifestSHA512; + return result; + } + } + + public boolean isEnabled() { + return enablePackages; + } + + public void handleZkErr(Exception e) { + log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e)); + } + + public boolean isJarInuse(String path) { + Packages pkg = null; + try { + pkg = readPkgsFromZk(null, null); + } catch (KeeperException.NoNodeException nne) { + return false; + } catch (InterruptedException | KeeperException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } + for (List vers : pkg.packages.values()) { + for (PkgVersion ver : vers) { + if (ver.files.contains(path)) { + return true; + } + } + } + return false; + } +} diff --git a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java index f3a86dd55d7b..98593a355a25 100644 --- a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java @@ -53,9 +53,9 @@ public class SolrPackageLoader implements Closeable { private final CoreContainer coreContainer; private final Map packageClassLoaders = new ConcurrentHashMap<>(); - private PackageAPI.Packages myCopy = new PackageAPI.Packages(); + private PackageStore.Packages myCopy = new PackageStore.Packages(); - private PackageAPI packageAPI; + private PackageStore packageStore; public Optional getPackageVersion(String pkg, String version) { SolrPackage p = packageClassLoaders.get(pkg); @@ -65,12 +65,12 @@ public Optional getPackageVersion(String pkg, String versio public SolrPackageLoader(CoreContainer coreContainer) { this.coreContainer = coreContainer; - packageAPI = new PackageAPI(coreContainer, this); + packageStore = new PackageStore(coreContainer, this); refreshPackageConf(); } - public PackageAPI getPackageAPI() { - return packageAPI; + public PackageStore getPackageStore() { + return packageStore; } public SolrPackage getPackage(String key) { @@ -83,12 +83,12 @@ public Map getPackages() { public void refreshPackageConf() { log.debug( - "{} updated to version {}", ZkStateReader.SOLR_PKGS_PATH, packageAPI.pkgs.znodeVersion); + "{} updated to version {}", ZkStateReader.SOLR_PKGS_PATH, packageStore.pkgs.znodeVersion); List updated = new ArrayList<>(); - Map> modified = getModified(myCopy, packageAPI.pkgs); + Map> modified = getModified(myCopy, packageStore.pkgs); - for (Map.Entry> e : modified.entrySet()) { + for (Map.Entry> e : modified.entrySet()) { if (e.getValue() != null) { SolrPackage p = packageClassLoaders.get(e.getKey()); if (e.getValue() != null && p == null) { @@ -109,14 +109,14 @@ public void refreshPackageConf() { for (SolrCore core : coreContainer.getCores()) { core.getPackageListeners().packagesUpdated(updated); } - myCopy = packageAPI.pkgs; + myCopy = packageStore.pkgs; } - public Map> getModified( - PackageAPI.Packages old, PackageAPI.Packages newPkgs) { - Map> changed = new HashMap<>(); - for (Map.Entry> e : newPkgs.packages.entrySet()) { - List versions = old.packages.get(e.getKey()); + public Map> getModified( + PackageStore.Packages old, PackageStore.Packages newPkgs) { + Map> changed = new HashMap<>(); + for (Map.Entry> e : newPkgs.packages.entrySet()) { + List versions = old.packages.get(e.getKey()); if (versions != null) { if (!Objects.equals(e.getValue(), versions)) { if (log.isInfoEnabled()) { @@ -172,8 +172,8 @@ public Set allVersions() { return myVersions.keySet(); } - private synchronized void updateVersions(List modified) { - for (PackageAPI.PkgVersion v : modified) { + private synchronized void updateVersions(List modified) { + for (PackageStore.PkgVersion v : modified) { Version version = myVersions.get(v.version); if (version == null) { log.info( @@ -194,7 +194,7 @@ private synchronized void updateVersions(List modified) { } Set newVersions = new HashSet<>(); - for (PackageAPI.PkgVersion v : modified) { + for (PackageStore.PkgVersion v : modified) { newVersions.add(v.version); } for (String s : new HashSet<>(myVersions.keySet())) { @@ -258,7 +258,7 @@ public class Version implements MapWriter, Closeable { private final SolrPackage parent; private SolrResourceLoader loader; - private final PackageAPI.PkgVersion version; + private final PackageStore.PkgVersion version; @Override public void writeMap(EntryWriter ew) throws IOException { @@ -266,7 +266,7 @@ public void writeMap(EntryWriter ew) throws IOException { version.writeMap(ew); } - Version(SolrPackage parent, PackageAPI.PkgVersion v) { + Version(SolrPackage parent, PackageStore.PkgVersion v) { this.parent = parent; this.version = v; List paths = new ArrayList<>(); @@ -293,7 +293,7 @@ public String getVersion() { return version.version; } - public PackageAPI.PkgVersion getPkgVersion() { + public PackageStore.PkgVersion getPkgVersion() { return version.copy(); } diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java index 06593f4b3ab9..4d451c17eaf3 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java +++ b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java @@ -58,8 +58,8 @@ import org.apache.solr.filestore.ClusterFileStore; import org.apache.solr.filestore.TestDistribFileStore; import org.apache.solr.filestore.TestDistribFileStore.Fetcher; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.solr.pkg.TestPackages; import org.apache.solr.request.SolrQueryRequest; @@ -93,7 +93,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return null; // only used to print meta information } diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java similarity index 89% rename from solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java rename to solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java index 8e3c1f4b72ca..b1a29c819464 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java @@ -28,12 +28,12 @@ import org.junit.Test; /** - * Integration tests for the JAX-RS-based {@link PackageAPIJaxRs}. + * Integration tests for the JAX-RS-based {@link PackageAPI}. * *

Note: SolrJettyTestRule cannot be used here because the Package API requires ZooKeeper for its * cluster-level operations. A one-node SolrCloud cluster is used instead. */ -public class PackageAPIJaxRsTest extends SolrCloudTestCase { +public class PackageAPITest extends SolrCloudTestCase { @BeforeClass public static void setupCluster() throws Exception { @@ -56,7 +56,7 @@ public void testListPackagesReturnsResult() throws Exception { @Test public void testAddAndDeletePackageVersion() throws Exception { - String FILE1 = "/jaxrstestpkg/runtimelibs.jar"; + String FILE1 = "/pkgapitestpkg/runtimelibs.jar"; // Upload a key and a signed jar file to the filestore byte[] derFile = @@ -71,7 +71,7 @@ public void testAddAndDeletePackageVersion() throws Exception { try (HttpSolrClient client = new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { // Add a package version via POST /cluster/package/{name}/versions - PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("jaxrstestpkg"); + PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("pkgapitestpkg"); addRequest.setVersion("1.0"); addRequest.setFiles(List.of(FILE1)); addRequest.process(client); @@ -81,29 +81,29 @@ public void testAddAndDeletePackageVersion() throws Exception { assertNotNull("Expected non-null list response", listResponse); assertNotNull("Expected non-null result", listResponse.result); assertNotNull( - "Expected jaxrstestpkg in packages", listResponse.result.packages.get("jaxrstestpkg")); + "Expected pkgapitestpkg in packages", listResponse.result.packages.get("pkgapitestpkg")); assertFalse( "Expected at least one version", - listResponse.result.packages.get("jaxrstestpkg").isEmpty()); + listResponse.result.packages.get("pkgapitestpkg").isEmpty()); // Verify GET /cluster/package/{name} returns only this package PackagesResponse getByNameResponse = - new PackageApi.GetPackage("jaxrstestpkg").process(client); + new PackageApi.GetPackage("pkgapitestpkg").process(client); assertNotNull("Expected non-null get-by-name response", getByNameResponse); assertNotNull("Expected non-null result from get-by-name", getByNameResponse.result); assertNotNull( - "Expected jaxrstestpkg in get-by-name response", - getByNameResponse.result.packages.get("jaxrstestpkg")); + "Expected pkgapitestpkg in get-by-name response", + getByNameResponse.result.packages.get("pkgapitestpkg")); // Delete the package version via DELETE /cluster/package/{name}/versions/{version} - new PackageApi.DeletePackageVersion("jaxrstestpkg", "1.0").process(client); + new PackageApi.DeletePackageVersion("pkgapitestpkg", "1.0").process(client); // Verify it's deleted PackagesResponse listAfterDelete = new PackageApi.ListPackages().process(client); assertNotNull("Expected non-null list response after delete", listAfterDelete); assertNotNull("Expected non-null result after delete", listAfterDelete.result); // After deleting the only version, the package entry should be empty or absent - List versionsAfterDelete = listAfterDelete.result.packages.get("jaxrstestpkg"); + List versionsAfterDelete = listAfterDelete.result.packages.get("pkgapitestpkg"); assertTrue( "Expected no versions after delete", versionsAfterDelete == null || versionsAfterDelete.isEmpty()); @@ -137,7 +137,7 @@ public void testRefreshNonExistentPackage() throws Exception { // Try to refresh a non-existent package. // Note: JacksonDataBindResponseParser doesn't expose errors as RemoteSolrException; // instead, the error is in the response body's 'error' field. - var response = new PackageApi.RefreshPackage("nonexistentpkg_jaxrs").process(client); + var response = new PackageApi.RefreshPackage("nonexistentpkg_test").process(client); assertNotNull("Expected error in response for non-existent package", response.error); assertEquals("Expected 400 for non-existent package", 400, (int) response.error.code); assertTrue( From d71f19933785eea0cf67781b85f42416300e7cbb Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sun, 1 Mar 2026 11:56:51 -0500 Subject: [PATCH 07/21] Additional migrations from the old "command" style to the new RESTful style --- .../solr/packagemanager/PackageManager.java | 27 ++--------- .../packagemanager/RepositoryManager.java | 33 ++++--------- .../solr/handler/TestContainerPlugin.java | 23 ++++----- .../pkg/PackageStoreSchemaPluginsTest.java | 22 ++------- .../solrj/request/beans/PackagePayload.java | 47 ------------------- 5 files changed, 27 insertions(+), 125 deletions(-) delete mode 100644 solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/PackagePayload.java diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java index 943c9248a62b..4dc807b44e2e 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java @@ -51,10 +51,8 @@ import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.client.solrj.request.GenericV2SolrRequest; -import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.client.solrj.request.beans.PackagePayload; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.beans.PluginMeta; -import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; @@ -152,20 +150,9 @@ public void uninstall(String packageName, String version) // Delete the package by calling the Package API and remove the Jar printGreen("Executing Package API to remove this package..."); - PackagePayload.DelVersion del = new PackagePayload.DelVersion(); - del.version = version; - del.pkg = packageName; - - V2Request req = - new V2Request.Builder(PackageUtils.PACKAGE_PATH) - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("delete", del)) - .build(); - try { - V2Response resp = req.process(solrClient); - printGreen("Response: " + resp.jsonStr()); + new PackageApi.DeletePackageVersion(packageName, version).process(solrClient); + printGreen("Package version deleted from Package API."); } catch (SolrServerException | IOException e) { throw new SolrException(ErrorCode.BAD_REQUEST, e); } @@ -467,10 +454,7 @@ private Pair, List> deployCollectionPackage( // If updating, refresh the package version for this to take effect if (isUpdate || pegToLatest) { try { - SolrCLI.postJsonToSolr( - solrClient, - PackageUtils.PACKAGE_PATH, - "{\"refresh\": \"" + packageInstance.name + "\"}"); + new PackageApi.RefreshPackage(packageInstance.name).process(solrClient); } catch (Exception ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); } @@ -1082,8 +1066,7 @@ public void undeploy( solrClient, PackageUtils.getCollectionParamsPath(collection), "{set: {PKG_VERSIONS: {" + packageName + ": null}}}"); - SolrCLI.postJsonToSolr( - solrClient, PackageUtils.PACKAGE_PATH, "{\"refresh\": \"" + packageName + "\"}"); + new PackageApi.RefreshPackage(packageName).process(solrClient); } catch (Exception ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); } diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java index 5ae49bd02a39..3a99673b1531 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java @@ -40,20 +40,15 @@ import org.apache.solr.cli.SolrCLI; import org.apache.solr.client.api.util.SolrVersion; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.FileStoreApi; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.GenericV2SolrRequest; -import org.apache.solr.client.solrj.request.RequestWriter; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.SystemInfoRequest; -import org.apache.solr.client.solrj.request.beans.PackagePayload; import org.apache.solr.client.solrj.response.SystemInfoResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Utils; import org.apache.solr.filestore.ClusterFileStore; import org.apache.solr.packagemanager.SolrPackage.Artifact; @@ -226,10 +221,9 @@ private boolean installPackage(String packageName, String version) throws SolrEx // Call Package API to add this version of the package printGreen("Executing Package API to register this package..."); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = version; - add.pkg = packageName; - add.files = + PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion(packageName); + addRequest.setVersion(version); + addRequest.setFiles( downloaded.stream() .map( file -> @@ -239,21 +233,12 @@ private boolean installPackage(String packageName, String version) throws SolrEx packageName, version, file.getFileName().toString())) - .collect(Collectors.toList()); - add.manifest = "/package/" + packageName + "/" + version + "/manifest.json"; - add.manifestSHA512 = manifestSHA512; - - GenericSolrRequest request = - new GenericV2SolrRequest(SolrRequest.METHOD.POST, PackageUtils.PACKAGE_PATH) { - @Override - public RequestWriter.ContentWriter getContentWriter(String expectedType) { - return new RequestWriter.StringPayloadContentWriter( - "{add:" + add.jsonStr() + "}", "application/json"); - } - }; + .collect(Collectors.toList())); + addRequest.setManifest("/package/" + packageName + "/" + version + "/manifest.json"); + addRequest.setManifestSHA512(manifestSHA512); try { - NamedList resp = solrClient.request(request); - printGreen("Response: " + resp.jsonStr()); + addRequest.process(solrClient); + printGreen("Package version registered successfully."); } catch (SolrServerException | IOException e) { throw new SolrException(ErrorCode.BAD_REQUEST, e); } diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java index 4d451c17eaf3..3b312785e175 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java +++ b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java @@ -42,8 +42,8 @@ import org.apache.solr.client.solrj.RemoteSolrException; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.client.solrj.request.beans.PackagePayload; import org.apache.solr.client.solrj.request.beans.PluginMeta; import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.cloud.ClusterSingleton; @@ -309,16 +309,9 @@ public void testApiFromPackage() throws Exception { // We have two versions of the plugin in 2 different jar files. they are already uploaded to // the package store listener.reset(); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = "1.0"; - add.pkg = "mypkg"; - add.files = singletonList(FILE1); - V2Request addPkgVersionReq = - new V2Request.Builder("/cluster/package") - .forceV2(forceV2) - .POST() - .withPayload(singletonMap("add", add)) - .build(); + PackageApi.AddPackageVersion addPkgVersionReq = new PackageApi.AddPackageVersion("mypkg"); + addPkgVersionReq.setVersion("1.0"); + addPkgVersionReq.setFiles(singletonList(FILE1)); addPkgVersionReq.process(cluster.getSolrClient()); assertTrue( "core package listeners did not notify", @@ -336,7 +329,7 @@ public void testApiFromPackage() throws Exception { PluginMeta plugin = new PluginMeta(); plugin.name = "myplugin"; plugin.klass = "mypkg:org.apache.solr.handler.MyPlugin"; - plugin.version = add.version; + plugin.version = "1.0"; final V2Request addPluginReq = postPlugin(singletonMap("add", plugin)); addPluginReq.process(cluster.getSolrClient()); version = phaser.awaitAdvanceInterruptibly(version, 10, TimeUnit.SECONDS); @@ -352,12 +345,12 @@ public void testApiFromPackage() throws Exception { TestDistribFileStore.assertResponseValues(invokePlugin, Map.of("/myplugin.version", "1.0")); // now let's upload the jar file for version 2.0 of the plugin - add.version = "2.0"; - add.files = singletonList(FILE2); + addPkgVersionReq.setVersion("2.0"); + addPkgVersionReq.setFiles(singletonList(FILE2)); addPkgVersionReq.process(cluster.getSolrClient()); // here the plugin version is updated - plugin.version = add.version; + plugin.version = "2.0"; postPlugin(singletonMap("update", plugin)).process(cluster.getSolrClient()); version = phaser.awaitAdvanceInterruptibly(version, 10, TimeUnit.SECONDS); diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java index 8faac9b68f9e..4091472b3331 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java @@ -27,10 +27,10 @@ import java.security.Signature; import java.util.Base64; import java.util.List; -import java.util.Map; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.response.SolrResponseBase; import org.apache.solr.cloud.SolrCloudTestCase; @@ -133,22 +133,10 @@ private void uploadPluginJar(String version, Path jarPath) throws Exception { } private void registerPackage(String version) throws Exception { - var packageRequest = - new V2Request.Builder("/cluster/package") - .POST() - .forceV2(true) - .withPayload( - Map.of( - "add", - Map.of( - "package", - "mypkg", - "version", - version, - "files", - List.of("/my-plugin/plugin-" + version + ".jar")))) - .build(); - processRequest(client, packageRequest); + var addRequest = new PackageApi.AddPackageVersion("mypkg"); + addRequest.setVersion(version); + addRequest.setFiles(List.of("/my-plugin/plugin-" + version + ".jar")); + processRequest(client, addRequest); } private void createCollection() throws Exception { diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/PackagePayload.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/PackagePayload.java deleted file mode 100644 index 831d87d809f7..000000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/PackagePayload.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.request.beans; - -import java.util.List; -import org.apache.solr.common.annotation.JsonProperty; -import org.apache.solr.common.util.ReflectMapWriter; - -/** Just a container class for POJOs used in Package APIs */ -public class PackagePayload { - public static class AddVersion implements ReflectMapWriter { - @JsonProperty(value = "package", required = true) - public String pkg; - - @JsonProperty(required = true) - public String version; - - @JsonProperty(required = true) - public List files; - - @JsonProperty public String manifest; - @JsonProperty public String manifestSHA512; - } - - public static class DelVersion implements ReflectMapWriter { - @JsonProperty(value = "package", required = true) - public String pkg; - - @JsonProperty(required = true) - public String version; - } -} From 8994e034b79368b8b9c12d4c4ed0953f85123210 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sun, 1 Mar 2026 12:53:51 -0500 Subject: [PATCH 08/21] fix test --- .../test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java index 4091472b3331..25f0e481cd72 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java @@ -136,7 +136,7 @@ private void registerPackage(String version) throws Exception { var addRequest = new PackageApi.AddPackageVersion("mypkg"); addRequest.setVersion(version); addRequest.setFiles(List.of("/my-plugin/plugin-" + version + ".jar")); - processRequest(client, addRequest); + addRequest.process(client); } private void createCollection() throws Exception { From 199b0f312bdb46536e3bede12004576b6f8a2969 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Tue, 3 Mar 2026 08:16:45 -0500 Subject: [PATCH 09/21] doc change --- changelog/unreleased/migrate-packageapi-to-jax-rs.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/unreleased/migrate-packageapi-to-jax-rs.yml diff --git a/changelog/unreleased/migrate-packageapi-to-jax-rs.yml b/changelog/unreleased/migrate-packageapi-to-jax-rs.yml new file mode 100644 index 000000000000..62157390e348 --- /dev/null +++ b/changelog/unreleased/migrate-packageapi-to-jax-rs.yml @@ -0,0 +1,7 @@ +title: Migrate PackageAPI to JAX-RS. PackageAPI now has OpenAPI and SolrJ support. +type: changed +authors: + - name: Eric Pugh +links: +- name: PR#4178 + url: https://github.com/apache/solr/pull/4178 From ae94dd366c051d8b26e6ca9c85e847120262e6d6 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 07:43:48 -0400 Subject: [PATCH 10/21] Updates from merging in latest code from main. --- .../solr/handler/component/SearchHandler.java | 2 +- .../packagemanager/RepositoryManager.java | 6 +- .../java/org/apache/solr/pkg/PackageAPI.java | 3 +- .../apache/solr/pkg/PackagePluginHolder.java | 2 +- .../solr/handler/TestContainerPlugin.java | 2 +- .../org/apache/solr/pkg/PackageAPITest.java | 59 ++++++++++--------- .../org/apache/solr/pkg/TestPackages.java | 14 ++--- 7 files changed, 43 insertions(+), 45 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java index c9f05e0c2563..20711a94b9f8 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java @@ -201,7 +201,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return Map.of(); } diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java index c74a80181422..6867d17b3653 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java @@ -43,12 +43,10 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.FileStoreApi; import org.apache.solr.client.solrj.request.PackageApi; -import org.apache.solr.client.solrj.request.SystemInfoRequest; -import org.apache.solr.client.solrj.response.SystemInfoResponse; +import org.apache.solr.client.solrj.request.SystemApi; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.Utils; import org.apache.solr.filestore.ClusterFileStore; import org.apache.solr.packagemanager.SolrPackage.Artifact; @@ -162,7 +160,7 @@ private String getRepositoriesJson(SolrZkClient zkClient) /** * Install a given package and version from the available repositories to Solr. The various steps * for doing so are, briefly, a) find upload a manifest to package store, b) download the - * artifacts and upload to package store, c) call {@link PackageAPI} to register the package. + * artifacts and upload to package store, c) call {@link PackageManager} to register the package. */ private boolean installPackage(String packageName, String version) throws SolrException { SolrPackageInstance existingPlugin = packageManager.getPackageInstance(packageName, version); diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index 0e0318bd0505..239e6a546329 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -103,7 +102,7 @@ public PackagesResponse getPackage(String packageName) { // Filter to only the requested package if (response.result != null && response.result.packages != null) { final var pkgVersions = response.result.packages.get(packageName); - response.result.packages = Collections.singletonMap(packageName, pkgVersions); + response.result.packages = Map.of(packageName, pkgVersions); } return response; } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java index 419cb033835f..b68f275232fa 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java @@ -58,7 +58,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return Map.of(info.cName.original, pkgVersion.getPkgVersion()); } diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java index 82b5847ea164..950842555669 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java +++ b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java @@ -310,7 +310,7 @@ public void testApiFromPackage() throws Exception { listener.reset(); PackageApi.AddPackageVersion addPkgVersionReq = new PackageApi.AddPackageVersion("mypkg"); addPkgVersionReq.setVersion("1.0"); - addPkgVersionReq.setFiles(singletonList(FILE1)); + addPkgVersionReq.setFiles(List.of(FILE1)); addPkgVersionReq.process(cluster.getSolrClient()); assertTrue( "core package listeners did not notify", diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java b/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java index b1a29c819464..aaadebc00ee3 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java +++ b/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java @@ -20,10 +20,12 @@ import java.util.List; import org.apache.solr.client.api.model.PackagesResponse; -import org.apache.solr.client.solrj.apache.HttpSolrClient; +import org.apache.solr.client.solrj.RemoteSolrException; +import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.filestore.ClusterFileStore; +import org.apache.solr.filestore.TestDistribFileStore; import org.junit.BeforeClass; import org.junit.Test; @@ -46,8 +48,9 @@ public static void setupCluster() throws Exception { @Test public void testListPackagesReturnsResult() throws Exception { - try (HttpSolrClient client = - new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { PackagesResponse response = new PackageApi.ListPackages().process(client); assertNotNull("Expected non-null response from GET /cluster/package", response); assertNotNull("Expected 'result' field in GET /cluster/package response", response.result); @@ -59,8 +62,7 @@ public void testAddAndDeletePackageVersion() throws Exception { String FILE1 = "/pkgapitestpkg/runtimelibs.jar"; // Upload a key and a signed jar file to the filestore - byte[] derFile = - org.apache.solr.filestore.TestDistribFileStore.readFile("cryptokeys/pub_key512.der"); + byte[] derFile = TestDistribFileStore.readFile("cryptokeys/pub_key512.der"); uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); TestPackages.postFileAndWait( cluster, @@ -68,8 +70,9 @@ public void testAddAndDeletePackageVersion() throws Exception { FILE1, "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - try (HttpSolrClient client = - new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { // Add a package version via POST /cluster/package/{name}/versions PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("pkgapitestpkg"); addRequest.setVersion("1.0"); @@ -111,38 +114,36 @@ public void testAddAndDeletePackageVersion() throws Exception { } @Test - public void testAddPackageVersionValidatesFiles() throws Exception { - try (HttpSolrClient client = - new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { - // Try to add a package version with a non-existent file. - // Note: JacksonDataBindResponseParser doesn't expose errors as RemoteSolrException; - // instead, the error is in the response body's 'error' field. + public void testAddPackageVersionValidatesFiles() { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("testpkg_invalid"); addRequest.setVersion("1.0"); addRequest.setFiles(List.of("/nonexistent/file.jar")); - var response = addRequest.process(client); - assertNotNull("Expected error in response for non-existent file", response.error); - assertEquals("Expected 400 for non-existent file", 400, (int) response.error.code); + RemoteSolrException ex = + expectThrows(RemoteSolrException.class, () -> addRequest.process(client)); + assertEquals("Expected 400 for non-existent file", 400, ex.code()); assertTrue( - "Expected error message to mention the file", - response.error.msg.contains("No such file")); + "Expected error message to mention the file: " + ex.getMessage(), + ex.getMessage().contains("No such file")); } } @Test - public void testRefreshNonExistentPackage() throws Exception { - try (HttpSolrClient client = - new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { - // Try to refresh a non-existent package. - // Note: JacksonDataBindResponseParser doesn't expose errors as RemoteSolrException; - // instead, the error is in the response body's 'error' field. - var response = new PackageApi.RefreshPackage("nonexistentpkg_test").process(client); - assertNotNull("Expected error in response for non-existent package", response.error); - assertEquals("Expected 400 for non-existent package", 400, (int) response.error.code); + public void testRefreshNonExistentPackage() { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + RemoteSolrException ex = + expectThrows( + RemoteSolrException.class, + () -> new PackageApi.RefreshPackage("nonexistentpkg_test").process(client)); + assertEquals("Expected 400 for non-existent package", 400, ex.code()); assertTrue( - "Expected error message to mention the package", - response.error.msg.contains("No such package")); + "Expected error message to mention the package: " + ex.getMessage(), + ex.getMessage().contains("No such package")); } } } diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index a7d1894bb0c4..249fd43ef725 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -134,7 +134,7 @@ public void testCoreReloadingPlugin() throws Exception { AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); add.version = "1.0"; - add.files = Arrays.asList(new String[] {FILE1}); + add.files = List.of(FILE1); V2Request req = new V2Request.Builder("/cluster/package/mypkg/versions") .forceV2(true) @@ -214,7 +214,7 @@ public void testPluginLoading() throws Exception { AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); add.version = "1.0"; - add.files = Arrays.asList(new String[] {FILE1, URP1, EXPR1}); + add.files = List.of(FILE1, URP1, EXPR1); V2Request req = new V2Request.Builder("/cluster/package/mypkg/versions") .forceV2(true) @@ -343,7 +343,7 @@ public void testPluginLoading() throws Exception { "P/ptFXRvQMd4oKPvadSpd+A9ffwY3gcex5GVFVRy3df0/OF8XT5my8rQz7FZva+2ORbWxdXS8NKwNrbPVHLGXw=="); // add the version using package API add.version = "1.1"; - add.files = Arrays.asList(new String[] {FILE2, URP2, EXPR1}); + add.files = List.of(FILE2, URP2, EXPR1); req.process(cluster.getSolrClient()); verifyComponent( @@ -372,7 +372,7 @@ public void testPluginLoading() throws Exception { "a400n4T7FT+2gM0SC6+MfSOExjud8MkhTSFylhvwNjtWwUgKdPFn434Wv7Qc4QEqDVLhQoL3WqYtQmLPti0G4Q=="); add.version = "2.1"; - add.files = Arrays.asList(new String[] {FILE3, URP2, EXPR1}); + add.files = List.of(FILE3, URP2, EXPR1); req.process(cluster.getSolrClient()); // now let's verify that the classes are updated @@ -445,7 +445,7 @@ public RequestWriter.ContentWriter getContentWriter(String expectedType) { }.setRequiresCollection(true).process(cluster.getSolrClient()); add.version = "2.1"; - add.files = Arrays.asList(new String[] {FILE3, URP2, EXPR1}); + add.files = List.of(FILE3, URP2, EXPR1); req.process(cluster.getSolrClient()); // the collections mypkg is set to use version 1.1 @@ -637,7 +637,7 @@ public void testAPI() throws Exception { // this time we are adding the second version of the package (0.13) add.version = "0.13"; - add.files = Collections.singletonList(FILE3); + add.files = List.of(FILE3); // this request should succeed req.process(cluster.getSolrClient()); @@ -888,7 +888,7 @@ public BaseWhitespaceTokenizerFactory(Map args) { } /* - //copy the jav files to a package and then run the main method + //copy the java files to a package and then run the main method public static void main(String[] args) throws Exception { persistZip("/tmp/x.jar", MyPatternReplaceCharFilterFactory.class, MyTextField.class, MyWhitespaceTokenizerFactory.class); }*/ From 62212c3930f4f583449db014f372c517a0b8c4f5 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 07:49:45 -0400 Subject: [PATCH 11/21] PackageAPI and PackageApis super confusing. Rename PackageAPI to ClusterPackage to follow naming pattern of other V2 apis. --- changelog/unreleased/migrate-packageapi-to-jax-rs.yml | 2 +- solr/core/src/java/org/apache/solr/core/CoreContainer.java | 4 ++-- .../solr/pkg/{PackageAPI.java => ClusterPackage.java} | 6 +++--- .../pkg/{PackageAPITest.java => ClusterPackageTest.java} | 4 ++-- solr/core/src/test/org/apache/solr/pkg/TestPackages.java | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) rename solr/core/src/java/org/apache/solr/pkg/{PackageAPI.java => ClusterPackage.java} (98%) rename solr/core/src/test/org/apache/solr/pkg/{PackageAPITest.java => ClusterPackageTest.java} (98%) diff --git a/changelog/unreleased/migrate-packageapi-to-jax-rs.yml b/changelog/unreleased/migrate-packageapi-to-jax-rs.yml index 62157390e348..f2d28ad93aba 100644 --- a/changelog/unreleased/migrate-packageapi-to-jax-rs.yml +++ b/changelog/unreleased/migrate-packageapi-to-jax-rs.yml @@ -1,4 +1,4 @@ -title: Migrate PackageAPI to JAX-RS. PackageAPI now has OpenAPI and SolrJ support. +title: Migrate PackageAPI to JAX-RS (now ClusterPackage). The package management API now has OpenAPI and SolrJ support. type: changed authors: - name: Eric Pugh diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 0325b920415e..60aff6f7dac2 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -135,7 +135,7 @@ import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.metrics.otel.OtelUnit; -import org.apache.solr.pkg.PackageAPI; +import org.apache.solr.pkg.ClusterPackage; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequestBase; @@ -848,7 +848,7 @@ private void loadInternal() { registerV2Api(ClusterFileStore.class); packageLoader = new SolrPackageLoader(this); - registerV2Api(PackageAPI.class); + registerV2Api(ClusterPackage.class); registerV2Api(ZookeeperRead.class); } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java similarity index 98% rename from solr/core/src/java/org/apache/solr/pkg/PackageAPI.java rename to solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java index 239e6a546329..4a58cb7cb96d 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -54,7 +54,7 @@ * * @see PackageApis */ -public class PackageAPI extends JerseyResource implements PackageApis { +public class ClusterPackage extends JerseyResource implements PackageApis { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final int SYNC_MAX_RETRIES = 10; @@ -65,7 +65,7 @@ public class PackageAPI extends JerseyResource implements PackageApis { private final SolrQueryResponse solrQueryResponse; @Inject - public PackageAPI( + public ClusterPackage( CoreContainer coreContainer, SolrQueryRequest solrQueryRequest, SolrQueryResponse solrQueryResponse) { @@ -333,7 +333,7 @@ private static PackagesResponse.PackageData toPackageData(PackageStore.Packages Map.Entry::getKey, e -> e.getValue().stream() - .map(PackageAPI::toPkgVersionResponse) + .map(ClusterPackage::toPkgVersionResponse) .collect(Collectors.toList()))); return data; } diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java similarity index 98% rename from solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java rename to solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java index aaadebc00ee3..dcbde47a7c59 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java +++ b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java @@ -30,12 +30,12 @@ import org.junit.Test; /** - * Integration tests for the JAX-RS-based {@link PackageAPI}. + * Integration tests for the JAX-RS-based {@link ClusterPackage}. * *

Note: SolrJettyTestRule cannot be used here because the Package API requires ZooKeeper for its * cluster-level operations. A one-node SolrCloud cluster is used instead. */ -public class PackageAPITest extends SolrCloudTestCase { +public class ClusterPackageTest extends SolrCloudTestCase { @BeforeClass public static void setupCluster() throws Exception { diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 249fd43ef725..22850ecbd139 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -88,7 +88,7 @@ import org.junit.Before; import org.junit.Test; -@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG") +@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.ClusterPackage=DEBUG") public class TestPackages extends SolrCloudTestCase { @Before From 30ab7ccf3a9ac7170bfed1cd7c3195e7132fec8d Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 12:30:38 -0400 Subject: [PATCH 12/21] Fix up expectedVersion property that was missing. Add test, revamp description. --- .../solr/client/api/endpoint/PackageApis.java | 11 ++++-- .../org/apache/solr/pkg/ClusterPackage.java | 12 +++++-- .../apache/solr/pkg/ClusterPackageTest.java | 34 +++++++++++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java index 9aeb40119d76..cf3a50f8caff 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java @@ -38,7 +38,9 @@ public interface PackageApis { summary = "List all packages registered in this Solr cluster.", tags = {"package"}) PackagesResponse listPackages( - @Parameter(description = "If provided, the named package is refreshed on this node.") + @Parameter( + description = + "Name of a package to refresh on the receiving node only (reloads its listeners from ZooKeeper). Used internally for inter-node fan-out by the refresh endpoint; not normally invoked directly.") @QueryParam("refreshPackage") String refreshPackage, @Parameter( @@ -55,7 +57,12 @@ PackagesResponse listPackages( PackagesResponse getPackage( @Parameter(description = "The name of the package.", required = true) @PathParam("packageName") - String packageName); + String packageName, + @Parameter( + description = + "If provided, the node waits until its package data matches this ZooKeeper version before responding.") + @QueryParam("expectedVersion") + Integer expectedVersion); @POST @Path("/{packageName}/versions") diff --git a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java index 4a58cb7cb96d..a091555e664f 100644 --- a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -95,14 +95,20 @@ public PackagesResponse listPackages(String refreshPackage, Integer expectedVers @Override @PermissionName(PACKAGE_READ_PERM) - public PackagesResponse getPackage(String packageName) { + public PackagesResponse getPackage(String packageName, Integer expectedVersion) { PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + + if (expectedVersion != null) { + syncToVersion(packageStore, expectedVersion); + } + final var response = instantiateJerseyResponse(PackagesResponse.class); response.result = toPackageData(packageStore.pkgs); - // Filter to only the requested package + // Filter to only the requested package; if absent, return an empty packages map. if (response.result != null && response.result.packages != null) { final var pkgVersions = response.result.packages.get(packageName); - response.result.packages = Map.of(packageName, pkgVersions); + response.result.packages = + pkgVersions == null ? Map.of() : Map.of(packageName, pkgVersions); } return response; } diff --git a/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java index dcbde47a7c59..324520161a1b 100644 --- a/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java @@ -146,4 +146,38 @@ public void testRefreshNonExistentPackage() { ex.getMessage().contains("No such package")); } } + + @Test + public void testListPackagesAcceptsRefreshPackageQueryParam() throws Exception { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + PackageApi.ListPackages req = new PackageApi.ListPackages(); + req.setRefreshPackage("nonexistentpkg_refresh"); + // The inter-node refresh signal calls notifyListeners on this node and returns OK even when + // the package doesn't exist; main's behavior is identical. + PackagesResponse response = req.process(client); + assertNotNull(response); + } + } + + @Test + public void testListAndGetPackageAcceptExpectedVersionQueryParam() throws Exception { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + // expectedVersion at-or-below current is a no-op in syncToVersion and must return promptly. + PackageApi.ListPackages listReq = new PackageApi.ListPackages(); + listReq.setExpectedVersion(0); + PackagesResponse listResponse = listReq.process(client); + assertNotNull(listResponse); + assertNotNull(listResponse.result); + + PackageApi.GetPackage getReq = new PackageApi.GetPackage("anything"); + getReq.setExpectedVersion(0); + PackagesResponse getResponse = getReq.process(client); + assertNotNull(getResponse); + assertNotNull(getResponse.result); + } + } } From 99d90e759a295d47f9d0d5e6bc1b2e6a6fe29ca9 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 12:31:08 -0400 Subject: [PATCH 13/21] Use our generated solrj code instead of hand crafting. --- .../solr/filestore/TestDistribFileStore.java | 30 +++++++------------ 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java index 8e7b5ab80ed0..4585696d7805 100644 --- a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java +++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java @@ -32,6 +32,7 @@ import java.util.concurrent.Callable; import java.util.function.Predicate; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.solr.client.api.model.UploadToFileStoreResponse; import org.apache.solr.client.solrj.RemoteSolrException; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; @@ -39,12 +40,9 @@ import org.apache.solr.client.solrj.request.FileStoreApi; import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.response.SimpleSolrResponse; -import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.NavigableObject; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.JavaBinCodec; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.SuppressForbidden; @@ -97,7 +95,7 @@ public void testFileStoreManagement() throws Exception { "/package/mypkg/v1.0/runtimelibs.jar", "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - NavigableObject rsp = + UploadToFileStoreResponse rsp = postFile( cluster.getSolrClient(), getFileContent("runtimecode/runtimelibs.jar.bin"), @@ -105,8 +103,7 @@ public void testFileStoreManagement() throws Exception { "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); assertTrue( - Objects.requireNonNullElse(rsp._getStr("message"), "") - .contains("File with same metadata exists ")); + Objects.requireNonNullElse(rsp.message, "").contains("File with same metadata exists ")); assertResponseValues( 10, @@ -344,22 +341,15 @@ public static void uploadKey(byte[] bytes, String path, MiniSolrCloudCluster clu false); } - public static NavigableObject postFile( + public static UploadToFileStoreResponse postFile( SolrClient client, ByteBuffer buffer, String name, String sig) throws SolrServerException, IOException { - String resource = "/cluster/filestore/files" + name; - ModifiableSolrParams params = new ModifiableSolrParams(); - params.add("sig", sig); - V2Response rsp = - new V2Request.Builder(resource) - .withMethod(SolrRequest.METHOD.PUT) - .withPayload(buffer) - .forceV2(true) - .withMimeType("application/octet-stream") - .withParams(params) - .build() - .process(client); - assertEquals(name, rsp.getResponse().get(CommonParams.FILE)); + final var uploadReq = new FileStoreApi.UploadFile(name, new ByteBufferInputStream(buffer)); + if (sig != null) { + uploadReq.setSig(List.of(sig)); + } + final UploadToFileStoreResponse rsp = uploadReq.process(client); + assertEquals(name, rsp.file); return rsp; } From 796757471054952b6e6f8e0265dc2b606a3c941b Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 12:34:03 -0400 Subject: [PATCH 14/21] tidy --- solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java index a091555e664f..3036985eecd0 100644 --- a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -107,8 +107,7 @@ public PackagesResponse getPackage(String packageName, Integer expectedVersion) // Filter to only the requested package; if absent, return an empty packages map. if (response.result != null && response.result.packages != null) { final var pkgVersions = response.result.packages.get(packageName); - response.result.packages = - pkgVersions == null ? Map.of() : Map.of(packageName, pkgVersions); + response.result.packages = pkgVersions == null ? Map.of() : Map.of(packageName, pkgVersions); } return response; } From 749173088ff17f8171b449ad2bb251799d038ee4 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 12:40:11 -0400 Subject: [PATCH 15/21] Small cleanups. --- solr/core/src/java/org/apache/solr/pkg/PackageStore.java | 6 +++--- .../src/java/org/apache/solr/pkg/SolrPackageLoader.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageStore.java b/solr/core/src/java/org/apache/solr/pkg/PackageStore.java index 77c32899c392..d51c8f56a2a7 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageStore.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageStore.java @@ -54,7 +54,7 @@ public class PackageStore { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static final String ERR_MSG = - "Package loading is not enabled , Start your nodes with -Dsolr.packages.enabled=true"; + "Package loading is not enabled, start your nodes with -Dsolr.packages.enabled=true"; final CoreContainer coreContainer; final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); @@ -126,7 +126,7 @@ Packages readPkgsFromZk(byte[] data, Stat stat) throws KeeperException, Interrup stat = new Stat(); data = coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, null, stat); } - Packages packages = null; + Packages packages; if (data == null || data.length == 0) { packages = new Packages(); } else { @@ -221,7 +221,7 @@ public void handleZkErr(Exception e) { } public boolean isJarInuse(String path) { - Packages pkg = null; + Packages pkg; try { pkg = readPkgsFromZk(null, null); } catch (KeeperException.NoNodeException nne) { diff --git a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java index c0b0eb288556..84fbea486af4 100644 --- a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java @@ -134,7 +134,7 @@ public Map> getModified( // some packages are deleted altogether for (String s : old.packages.keySet()) { if (!newPkgs.packages.keySet().contains(s)) { - log.info("Package: {} is removed althogether", s); + log.info("Package: {} is removed altogether", s); changed.put(s, null); } } @@ -330,7 +330,7 @@ static class PackageResourceLoader extends SolrResourceLoader { @Override public boolean addToCoreAware(T obj) { // do not do anything - // this class is not aware of a SolrCore and it is totally not tied to + // this class is not aware of a SolrCore, and it is totally not tied to // the lifecycle of SolrCore. So, this returns 'false' & it should be // taken care of by the caller return false; From deb2142d0dc49574650d1dfbb4b7c7bdd52bbefd Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 12:44:05 -0400 Subject: [PATCH 16/21] Now have a JIRA issue --- .../unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml | 7 +++++++ changelog/unreleased/migrate-packageapi-to-jax-rs.yml | 7 ------- 2 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml delete mode 100644 changelog/unreleased/migrate-packageapi-to-jax-rs.yml diff --git a/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml b/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml new file mode 100644 index 000000000000..67ac97f93f9c --- /dev/null +++ b/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml @@ -0,0 +1,7 @@ +title: The package management API now has OpenAPI and SolrJ support. +type: changed +authors: + - name: Eric Pugh +links: +- name: PR#4178 + url: https://github.com/apache/solr/pull/4178 diff --git a/changelog/unreleased/migrate-packageapi-to-jax-rs.yml b/changelog/unreleased/migrate-packageapi-to-jax-rs.yml deleted file mode 100644 index f2d28ad93aba..000000000000 --- a/changelog/unreleased/migrate-packageapi-to-jax-rs.yml +++ /dev/null @@ -1,7 +0,0 @@ -title: Migrate PackageAPI to JAX-RS (now ClusterPackage). The package management API now has OpenAPI and SolrJ support. -type: changed -authors: - - name: Eric Pugh -links: -- name: PR#4178 - url: https://github.com/apache/solr/pull/4178 From 2578d2670791fbe0c3b285f18907b2d01df14874 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 14:18:53 -0400 Subject: [PATCH 17/21] shockingly, we had these unused objects --- .../src/java/org/apache/solr/pkg/ClusterPackage.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java index 3036985eecd0..22c52d88bad7 100644 --- a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -43,8 +43,6 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.filestore.FileStoreUtils; import org.apache.solr.jersey.PermissionName; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,17 +59,10 @@ public class ClusterPackage extends JerseyResource implements PackageApis { private static final long SYNC_SLEEP_MS = 10L; private final CoreContainer coreContainer; - private final SolrQueryRequest solrQueryRequest; - private final SolrQueryResponse solrQueryResponse; @Inject - public ClusterPackage( - CoreContainer coreContainer, - SolrQueryRequest solrQueryRequest, - SolrQueryResponse solrQueryResponse) { + public ClusterPackage(CoreContainer coreContainer) { this.coreContainer = coreContainer; - this.solrQueryRequest = solrQueryRequest; - this.solrQueryResponse = solrQueryResponse; } @Override From bf46dc7c7bba1031a748ff3fe5ed31d71acd8bea Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 14:23:19 -0400 Subject: [PATCH 18/21] Don't need fancy instantiate method! --- .../src/java/org/apache/solr/pkg/ClusterPackage.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java index 22c52d88bad7..222e0a42c224 100644 --- a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -72,14 +72,14 @@ public PackagesResponse listPackages(String refreshPackage, Integer expectedVers if (refreshPackage != null) { packageStore.packageLoader.notifyListeners(refreshPackage); - return instantiateJerseyResponse(PackagesResponse.class); + return new PackagesResponse(); } if (expectedVersion != null) { syncToVersion(packageStore, expectedVersion); } - final var response = instantiateJerseyResponse(PackagesResponse.class); + final var response = new PackagesResponse(); response.result = toPackageData(packageStore.pkgs); return response; } @@ -93,7 +93,7 @@ public PackagesResponse getPackage(String packageName, Integer expectedVersion) syncToVersion(packageStore, expectedVersion); } - final var response = instantiateJerseyResponse(PackagesResponse.class); + final var response = new PackagesResponse(); response.result = toPackageData(packageStore.pkgs); // Filter to only the requested package; if absent, return an empty packages map. if (response.result != null && response.result.packages != null) { @@ -107,7 +107,7 @@ public PackagesResponse getPackage(String packageName, Integer expectedVersion) @PermissionName(PACKAGE_EDIT_PERM) public SolrJerseyResponse addPackageVersion( String packageName, AddPackageVersionRequestBody requestBody) { - final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + final var response = new SolrJerseyResponse(); PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); if (!packageStore.isEnabled()) { @@ -174,7 +174,7 @@ public SolrJerseyResponse addPackageVersion( @Override @PermissionName(PACKAGE_EDIT_PERM) public SolrJerseyResponse deletePackageVersion(String packageName, String version) { - final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + final var response = new SolrJerseyResponse(); PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); if (!packageStore.isEnabled()) { @@ -226,7 +226,7 @@ public SolrJerseyResponse deletePackageVersion(String packageName, String versio @Override @PermissionName(PACKAGE_EDIT_PERM) public SolrJerseyResponse refreshPackage(String packageName) { - final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + final var response = new SolrJerseyResponse(); PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(packageName); From a0bef9075d19671d40fd03747892382ace90a3c0 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 14:30:44 -0400 Subject: [PATCH 19/21] Use SolrJ methods where possible. --- .../solr/packagemanager/PackageManager.java | 7 +++--- .../org/apache/solr/pkg/ClusterPackage.java | 23 ++++--------------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java index 3dd95eece80b..4c7868e5aa7f 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java @@ -48,7 +48,7 @@ import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; -import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.FileStoreApi; import org.apache.solr.client.solrj.request.GenericV2SolrRequest; import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.beans.PluginMeta; @@ -163,9 +163,8 @@ public void uninstall(String packageName, String version) String.format(Locale.ROOT, "/package/%s/%s/%s", packageName, version, "manifest.json")); for (String filePath : filesToDelete) { DistribFileStore.deleteZKFileEntry(zkClient, filePath); - String path = "/api/cluster/filestore/files" + filePath; - printGreen("Deleting " + path); - solrClient.request(new GenericSolrRequest(SolrRequest.METHOD.DELETE, path)); + printGreen("Deleting " + filePath); + new FileStoreApi.DeleteFile(filePath).process(solrClient); } printGreen("Package uninstalled: " + packageName + ":" + version + ":-)"); diff --git a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java index 222e0a42c224..e7bdc0979374 100644 --- a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -33,12 +33,9 @@ import org.apache.solr.client.api.model.AddPackageVersionRequestBody; import org.apache.solr.client.api.model.PackagesResponse; import org.apache.solr.client.api.model.SolrJerseyResponse; -import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.response.JavaBinResponseParser; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.common.SolrException; -import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; import org.apache.solr.filestore.FileStoreUtils; @@ -237,13 +234,8 @@ public SolrJerseyResponse refreshPackage(String packageName) { // first refresh on the current node packageStore.packageLoader.notifyListeners(packageName); - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("refreshPackage", packageName); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); + final var request = new PackageApi.ListPackages(); + request.setRefreshPackage(packageName); for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { final var baseUrl = @@ -291,13 +283,8 @@ private void syncToVersion(PackageStore packageStore, int expectedVersion) { } private void notifyAllNodesToSync(int expectedVersion) { - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("expectedVersion", String.valueOf(expectedVersion)); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); + final var request = new PackageApi.ListPackages(); + request.setExpectedVersion(expectedVersion); for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { var baseUrl = coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); From ba61c1ba723541ac0d40c517229334009423d106 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 14:42:37 -0400 Subject: [PATCH 20/21] More conversion to SolrJ methods --- .../pkg/PackageStoreSchemaPluginsTest.java | 13 +- .../org/apache/solr/pkg/TestPackages.java | 123 ++++++------------ 2 files changed, 43 insertions(+), 93 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java index 25f0e481cd72..290943655ca7 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java @@ -30,8 +30,8 @@ import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.FileStoreApi; import org.apache.solr.client.solrj.request.PackageApi; -import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.response.SolrResponseBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.filestore.FileStoreAPI; @@ -123,13 +123,10 @@ private static String signature(byte[] content) throws Exception { private void uploadPluginJar(String version, Path jarPath) throws Exception { var pluginRequest = - new V2Request.Builder("/cluster/filestore/files/my-plugin/plugin-" + version + ".jar") - .PUT() - .withParams(params("sig", signature(Files.readAllBytes(jarPath)))) - .withPayload(Files.newInputStream(jarPath)) - .forceV2(true) - .build(); - processRequest(client, pluginRequest); + new FileStoreApi.UploadFile( + "/my-plugin/plugin-" + version + ".jar", Files.newInputStream(jarPath)); + pluginRequest.setSig(List.of(signature(Files.readAllBytes(jarPath)))); + pluginRequest.process(client); } private void registerPackage(String version) throws Exception { diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 22850ecbd139..05a8999a0973 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -43,13 +43,13 @@ import org.apache.lucene.analysis.pattern.PatternReplaceCharFilterFactory; import org.apache.lucene.util.ResourceLoader; import org.apache.lucene.util.ResourceLoaderAware; -import org.apache.solr.client.api.model.AddPackageVersionRequestBody; import org.apache.solr.client.solrj.RemoteSolrException; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.RequestWriter; import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.request.UpdateRequest; @@ -132,15 +132,9 @@ public void testCoreReloadingPlugin() throws Exception { FILE1, "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); - add.version = "1.0"; - add.files = List.of(FILE1); - V2Request req = - new V2Request.Builder("/cluster/package/mypkg/versions") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(add) - .build(); + PackageApi.AddPackageVersion req = new PackageApi.AddPackageVersion("mypkg"); + req.setVersion("1.0"); + req.setFiles(List.of(FILE1)); req.process(cluster.getSolrClient()); TestDistribFileStore.assertResponseValues( @@ -161,9 +155,9 @@ public void testCoreReloadingPlugin() throws Exception { cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); verifyComponent( - cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", "mypkg", add.version); + cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", "mypkg", "1.0"); - add.version = "2.0"; + req.setVersion("2.0"); req.process(cluster.getSolrClient()); TestDistribFileStore.assertResponseValues( 10, @@ -212,15 +206,9 @@ public void testPluginLoading() throws Exception { EXPR1, "ZOT11arAiPmPZYOHzqodiNnxO9pRyRozWZEBX8XGjU1/HJptFnZK+DI7eXnUtbNaMcbXE2Ze8hh4M/eGyhY8BQ=="); - AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); - add.version = "1.0"; - add.files = List.of(FILE1, URP1, EXPR1); - V2Request req = - new V2Request.Builder("/cluster/package/mypkg/versions") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(add) - .build(); + PackageApi.AddPackageVersion req = new PackageApi.AddPackageVersion("mypkg"); + req.setVersion("1.0"); + req.setFiles(List.of(FILE1, URP1, EXPR1)); req.process(cluster.getSolrClient()); @@ -342,8 +330,8 @@ public void testPluginLoading() throws Exception { URP2, "P/ptFXRvQMd4oKPvadSpd+A9ffwY3gcex5GVFVRy3df0/OF8XT5my8rQz7FZva+2ORbWxdXS8NKwNrbPVHLGXw=="); // add the version using package API - add.version = "1.1"; - add.files = List.of(FILE2, URP2, EXPR1); + req.setVersion("1.1"); + req.setFiles(List.of(FILE2, URP2, EXPR1)); req.process(cluster.getSolrClient()); verifyComponent( @@ -371,8 +359,8 @@ public void testPluginLoading() throws Exception { FILE3, "a400n4T7FT+2gM0SC6+MfSOExjud8MkhTSFylhvwNjtWwUgKdPFn434Wv7Qc4QEqDVLhQoL3WqYtQmLPti0G4Q=="); - add.version = "2.1"; - add.files = List.of(FILE3, URP2, EXPR1); + req.setVersion("2.1"); + req.setFiles(List.of(FILE3, URP2, EXPR1)); req.process(cluster.getSolrClient()); // now let's verify that the classes are updated @@ -402,11 +390,7 @@ public void testPluginLoading() throws Exception { assertEquals("Version 2", result.getResults().get(0).getFieldValue("TestVersionedURP.Ver_s")); - new V2Request.Builder("/cluster/package/mypkg/versions/1.0") - .withMethod(SolrRequest.METHOD.DELETE) - .forceV2(true) - .build() - .process(cluster.getSolrClient()); + new PackageApi.DeletePackageVersion("mypkg", "1.0").process(cluster.getSolrClient()); verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "queryResponseWriter", "json1", "mypkg", "2.1"); @@ -418,11 +402,7 @@ public void testPluginLoading() throws Exception { cluster.getSolrClient(), COLLECTION_NAME, "requestHandler", "/runtime", "mypkg", "2.1"); // now remove the highest version. So, it will roll back to the next highest one - new V2Request.Builder("/cluster/package/mypkg/versions/2.1") - .withMethod(SolrRequest.METHOD.DELETE) - .forceV2(true) - .build() - .process(cluster.getSolrClient()); + new PackageApi.DeletePackageVersion("mypkg", "2.1").process(cluster.getSolrClient()); verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "queryResponseWriter", "json1", "mypkg", "1.1"); @@ -444,8 +424,8 @@ public RequestWriter.ContentWriter getContentWriter(String expectedType) { } }.setRequiresCollection(true).process(cluster.getSolrClient()); - add.version = "2.1"; - add.files = List.of(FILE3, URP2, EXPR1); + req.setVersion("2.1"); + req.setFiles(List.of(FILE3, URP2, EXPR1)); req.process(cluster.getSolrClient()); // the collections mypkg is set to use version 1.1 @@ -471,11 +451,7 @@ public RequestWriter.ContentWriter getContentWriter(String expectedType) { // now, let's force every collection using 'mypkg' to refresh // so that it uses version 2.1 - new V2Request.Builder("/cluster/package/mypkg/refresh") - .withMethod(SolrRequest.METHOD.POST) - .forceV2(true) - .build() - .process(cluster.getSolrClient()); + new PackageApi.RefreshPackage("mypkg").process(cluster.getSolrClient()); verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "queryResponseWriter", "json1", "mypkg", "2.1"); @@ -581,15 +557,9 @@ public void testAPI() throws Exception { String FILE2 = "/mypkg/v.0.12/jar_b.jar"; String FILE3 = "/mypkg/v.0.13/jar_a.jar"; - AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); - add.version = "0.12"; - add.files = List.of(FILE1, FILE2); - V2Request req = - new V2Request.Builder("/cluster/package/test_pkg/versions") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(add) - .build(); + PackageApi.AddPackageVersion req = new PackageApi.AddPackageVersion("test_pkg"); + req.setVersion("0.12"); + req.setFiles(List.of(FILE1, FILE2)); // the files are not yet there. The command should fail with error saying "No such file" expectError(req, cluster.getSolrClient(), errPath, "No such file:"); @@ -597,7 +567,7 @@ public void testAPI() throws Exception { // post the jar file. No signature is sent postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1, null); - add.files = List.of(FILE1); + req.setFiles(List.of(FILE1)); expectError(req, cluster.getSolrClient(), errPath, FILE1 + " has no signature"); // now we upload the keys byte[] derFile = readFile("cryptokeys/pub_key512.der"); @@ -610,7 +580,7 @@ public void testAPI() throws Exception { "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); // with correct signature // after uploading the file, let's delete the keys to see if we get proper error message - add.files = List.of(FILE2); + req.setFiles(List.of(FILE2)); /*expectError(req, cluster.getSolrClient(), errPath, "ZooKeeper does not have any public keys");*/ @@ -636,8 +606,8 @@ public void testAPI() throws Exception { "j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA=="); // this time we are adding the second version of the package (0.13) - add.version = "0.13"; - add.files = List.of(FILE3); + req.setVersion("0.13"); + req.setFiles(List.of(FILE3)); // this request should succeed req.process(cluster.getSolrClient()); @@ -650,21 +620,14 @@ public void testAPI() throws Exception { Map.of(":packages:test_pkg[1]:version", "0.13", ":packages:test_pkg[1]:files[0]", FILE3)); // Now we will just delete one version - V2Request deleteReq = - new V2Request.Builder("/cluster/package/test_pkg/versions/0.1") - .forceV2(true) - .withMethod(SolrRequest.METHOD.DELETE) - .build(); + PackageApi.DeletePackageVersion deleteReq = + new PackageApi.DeletePackageVersion("test_pkg", "0.1"); // we are expecting an error expectError(deleteReq, cluster.getSolrClient(), errPath, "No such version:"); - new V2Request.Builder( - "/cluster/package/test_pkg/versions/0.12") // correct version. Should succeed - .forceV2(true) - .withMethod(SolrRequest.METHOD.DELETE) - .build() - .process(cluster.getSolrClient()); + // correct version. Should succeed + new PackageApi.DeletePackageVersion("test_pkg", "0.12").process(cluster.getSolrClient()); // Verify with ZK that the data is correct TestDistribFileStore.assertResponseValues( 1, @@ -756,15 +719,10 @@ public void testSchemaPlugins() throws Exception { "gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q=="); // upload package v1.0 - AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); - add.version = "1.0"; - add.files = Arrays.asList(FILE1, FILE2); - new V2Request.Builder("/cluster/package/schemapkg/versions") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(add) - .build() - .process(cluster.getSolrClient()); + PackageApi.AddPackageVersion addReq = new PackageApi.AddPackageVersion("schemapkg"); + addReq.setVersion("1.0"); + addReq.setFiles(Arrays.asList(FILE1, FILE2)); + addReq.process(cluster.getSolrClient()); TestDistribFileStore.assertResponseValues( 10, @@ -796,15 +754,9 @@ public void testSchemaPlugins() throws Exception { coreProvider.withCore(core -> schemas[0] = core.getLatestSchema()); // upload package v2.0 - add = new AddPackageVersionRequestBody(); - add.version = "2.0"; - add.files = Arrays.asList(FILE1, FILE2); - new V2Request.Builder("/cluster/package/schemapkg/versions") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(add) - .build() - .process(cluster.getSolrClient()); + addReq.setVersion("2.0"); + addReq.setFiles(Arrays.asList(FILE1, FILE2)); + addReq.process(cluster.getSolrClient()); TestDistribFileStore.assertResponseValues( 10, @@ -858,7 +810,8 @@ public static void postFileAndWait( cluster, path, Map.of(":files:" + path + ":sha512", sha512), false); } - private void expectError(V2Request req, SolrClient client, String errPath, String expectErrorMsg) + private void expectError( + SolrRequest req, SolrClient client, String errPath, String expectErrorMsg) throws IOException, SolrServerException { try { req.process(client); From 670562d9c91bd402fe31a2815fd69fe8c43f0e5c Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 15:02:23 -0400 Subject: [PATCH 21/21] Rework the tests. I am a bit on the fence about detailed docs. --- .../solr/filestore/TestDistribFileStore.java | 15 ++ .../solr/handler/TestContainerPlugin.java | 5 +- .../apache/solr/pkg/ClusterPackageTest.java | 2 +- ...=> ConfigsetPinnedPackageVersionTest.java} | 7 +- .../solr/pkg/PackageSchemaReloadTest.java | 166 ++++++++++++ .../org/apache/solr/pkg/TestPackages.java | 249 ++---------------- 6 files changed, 205 insertions(+), 239 deletions(-) rename solr/core/src/test/org/apache/solr/pkg/{PackageStoreSchemaPluginsTest.java => ConfigsetPinnedPackageVersionTest.java} (94%) create mode 100644 solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java index 4585696d7805..3d0f2fd74e0c 100644 --- a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java +++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java @@ -353,6 +353,21 @@ public static UploadToFileStoreResponse postFile( return rsp; } + /** + * Upload a file from the test classpath to the distributed file store under the given path, then + * wait until every node in the cluster reports the expected SHA-512 for that path. + */ + public static void postFileAndWait( + MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception { + ByteBuffer fileContent = getFileContent(fname); + @SuppressWarnings("ByteBufferBackingArray") // this is the result of a call to wrap() + String sha512 = DigestUtils.sha512Hex(fileContent.array()); + + postFile(cluster.getSolrClient(), fileContent, path, sig); + + checkAllNodesForFile(cluster, path, Map.of(":files:" + path + ":sha512", sha512), false); + } + /** * Read and return the contents of the file-like resource * diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java index 950842555669..4c35d7b94d7d 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java +++ b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java @@ -60,7 +60,6 @@ import org.apache.solr.pkg.PackageListeners; import org.apache.solr.pkg.PackageStore; import org.apache.solr.pkg.SolrPackageLoader; -import org.apache.solr.pkg.TestPackages; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.PermissionNameProvider; @@ -294,12 +293,12 @@ public void testApiFromPackage() throws Exception { byte[] derFile = readFile("cryptokeys/pub_key512.der"); uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); - TestPackages.postFileAndWait( + TestDistribFileStore.postFileAndWait( cluster, "runtimecode/containerplugin.v.1.jar.bin", FILE1, "pmrmWCDafdNpYle2rueAGnU2J6NYlcAey9mkZYbqh+5RdYo2Ln+llLF9voyRj+DDivK9GV1XdtKvD9rgCxlD7Q=="); - TestPackages.postFileAndWait( + TestDistribFileStore.postFileAndWait( cluster, "runtimecode/containerplugin.v.2.jar.bin", FILE2, diff --git a/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java index 324520161a1b..fef6cdb21be6 100644 --- a/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java @@ -64,7 +64,7 @@ public void testAddAndDeletePackageVersion() throws Exception { // Upload a key and a signed jar file to the filestore byte[] derFile = TestDistribFileStore.readFile("cryptokeys/pub_key512.der"); uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); - TestPackages.postFileAndWait( + TestDistribFileStore.postFileAndWait( cluster, "runtimecode/runtimelibs.jar.bin", FILE1, diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java b/solr/core/src/test/org/apache/solr/pkg/ConfigsetPinnedPackageVersionTest.java similarity index 94% rename from solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java rename to solr/core/src/test/org/apache/solr/pkg/ConfigsetPinnedPackageVersionTest.java index 290943655ca7..f5854b6e296f 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/ConfigsetPinnedPackageVersionTest.java @@ -39,7 +39,12 @@ import org.junit.Before; import org.junit.Test; -public class PackageStoreSchemaPluginsTest extends SolrCloudTestCase { +/** + * Verifies that a configset which pins a specific package version (via {@code params.json}) + * resolves classes from that exact version's jar — even when other versions of the same package are + * also registered. + */ +public class ConfigsetPinnedPackageVersionTest extends SolrCloudTestCase { private static final KeyPair KEY_PAIR; diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java new file mode 100644 index 000000000000..424cf4fc2f42 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.pkg; + +import static org.apache.solr.filestore.TestDistribFileStore.postFileAndWait; +import static org.apache.solr.filestore.TestDistribFileStore.readFile; +import static org.apache.solr.filestore.TestDistribFileStore.uploadKey; + +import java.util.Arrays; +import java.util.Map; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.PackageApi; +import org.apache.solr.client.solrj.request.V2Request; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.core.SolrCore; +import org.apache.solr.filestore.ClusterFileStore; +import org.apache.solr.filestore.TestDistribFileStore; +import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.IndexSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Verifies that bumping a package version that backs a schema plugin causes the affected core's + * schema to reload and pick up classes from the new package's classloader. + */ +public class PackageSchemaReloadTest extends SolrCloudTestCase { + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + System.setProperty("solr.packages.enabled", "true"); + configureCluster(4) + .withJettyConfig(jetty -> jetty.enableV2(true)) + .addConfig("conf1", configset("schema-package")) + .configure(); + } + + @After + @Override + public void tearDown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + } + super.tearDown(); + } + + @Test + public void testSchemaReloadOnPackageVersionBump() throws Exception { + String COLLECTION_NAME = "testSchemaLoadingColl"; + System.setProperty("managed.schema.mutable", "true"); + + IndexSchema[] schemas = new IndexSchema[2]; // tracks schemas for a selected core + + String FILE1 = "/schemapkg/schema-plugins.jar"; + byte[] derFile = readFile("cryptokeys/pub_key512.der"); + uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); + postFileAndWait( + cluster, + "runtimecode/schema-plugins.jar.bin", + FILE1, + "U+AdO/jgY3DtMpeFRGoTQk72iA5g/qjPvdQYPGBaXB5+ggcTZk4FoIWiueB0bwGJ8Mg3V/elxOqEbD2JR8R0tA=="); + + String FILE2 = "/schemapkg/payload-component.jar"; + postFileAndWait( + cluster, + "runtimecode/payload-component.jar.bin", + FILE2, + "gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q=="); + + // upload package v1.0 + PackageApi.AddPackageVersion addReq = new PackageApi.AddPackageVersion("schemapkg"); + addReq.setVersion("1.0"); + addReq.setFiles(Arrays.asList(FILE1, FILE2)); + addReq.process(cluster.getSolrClient()); + + TestDistribFileStore.assertResponseValues( + 10, + () -> + new V2Request.Builder("/cluster/package") + .withMethod(SolrRequest.METHOD.GET) + .build() + .process(cluster.getSolrClient()), + Map.of( + ":result:packages:schemapkg[0]:version", + "1.0", + ":result:packages:schemapkg[0]:files[0]", + FILE1)); + + CollectionAdminRequest.createCollection(COLLECTION_NAME, "conf1", 2, 2) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); + + // make note of the schema instance for one of the cores + SolrCore.Provider coreProvider = + cluster.getJettySolrRunners().stream() + .flatMap( + jetty -> + jetty.getCoreContainer().getAllCoreNames().stream() + .map(name -> new SolrCore.Provider(jetty.getCoreContainer(), name, null))) + .findFirst() + .orElseThrow(); + + coreProvider.withCore(core -> schemas[0] = core.getLatestSchema()); + + // upload package v2.0 + addReq.setVersion("2.0"); + addReq.setFiles(Arrays.asList(FILE1, FILE2)); + addReq.process(cluster.getSolrClient()); + + TestDistribFileStore.assertResponseValues( + 10, + () -> + new V2Request.Builder("/cluster/package") + .withMethod(SolrRequest.METHOD.GET) + .build() + .process(cluster.getSolrClient()), + Map.of( + ":result:packages:schemapkg[1]:version", + "2.0", + ":result:packages:schemapkg[1]:files[0]", + FILE1)); + + // even though package version 2.0 uses exactly the same files + // as version 1.0, the core schema should still reload, and + // the core should be associated with a different schema instance + TestDistribFileStore.assertResponseValues( + 10, + () -> { + coreProvider.withCore(core -> schemas[1] = core.getLatestSchema()); + return params("schemaReloaded", (schemas[0] != schemas[1]) ? "yes" : "no"); + }, + Map.of("schemaReloaded", "yes")); + + // after the reload, the custom field type class now comes from package v2.0 + String fieldTypeName = "myNewTextFieldWithAnalyzerClass"; + + FieldType fieldTypeV1 = schemas[0].getFieldTypeByName(fieldTypeName); + assertEquals("my.pkg.MyTextField", fieldTypeV1.getClass().getCanonicalName()); + + FieldType fieldTypeV2 = schemas[1].getFieldTypeByName(fieldTypeName); + assertEquals("my.pkg.MyTextField", fieldTypeV2.getClass().getCanonicalName()); + + assertNotEquals( + "my.pkg.MyTextField classes should be from different classloaders", + fieldTypeV1.getClass(), + fieldTypeV2.getClass()); + } +} diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 05a8999a0973..3dbcb0fef0fa 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -17,11 +17,10 @@ package org.apache.solr.pkg; -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; import static org.apache.solr.common.params.CommonParams.JAVABIN; import static org.apache.solr.common.params.CommonParams.WT; -import static org.apache.solr.core.TestSolrConfigHandler.getFileContent; import static org.apache.solr.filestore.TestDistribFileStore.checkAllNodesForFile; +import static org.apache.solr.filestore.TestDistribFileStore.postFileAndWait; import static org.apache.solr.filestore.TestDistribFileStore.readFile; import static org.apache.solr.filestore.TestDistribFileStore.uploadKey; @@ -30,15 +29,12 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; -import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.Callable; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -import org.apache.commons.codec.digest.DigestUtils; import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory; import org.apache.lucene.analysis.pattern.PatternReplaceCharFilterFactory; import org.apache.lucene.util.ResourceLoader; @@ -56,7 +52,6 @@ import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.util.ClientUtils; -import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.NavigableObject; import org.apache.solr.common.SolrInputDocument; @@ -64,7 +59,6 @@ import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.JavaBinCodec; import org.apache.solr.common.util.ReflectMapWriter; import org.apache.solr.common.util.Utils; import org.apache.solr.core.SolrCore; @@ -74,20 +68,31 @@ import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.schema.FieldType; -import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.QParser; import org.apache.solr.search.QParserPlugin; import org.apache.solr.security.AuthorizationContext; import org.apache.solr.util.LogLevel; import org.apache.solr.util.plugin.SolrCoreAware; -import org.apache.zookeeper.data.Stat; import org.eclipse.jetty.client.ContentResponse; import org.eclipse.jetty.client.HttpClient; import org.junit.After; import org.junit.Before; import org.junit.Test; +/** + * Multi-node tests covering how Solr consumes packages at runtime: plugin loading via + * {@link SolrPackageLoader} / {@link PackagePluginHolder}, core reload on package version change, + * cross-node refresh fan-out, and package-file download to newly-joined nodes. + * + *

API-contract tests for the {@link ClusterPackage} JAX-RS endpoints live in {@link + * ClusterPackageTest}. Schema-reload-on-version-bump lives in {@link PackageSchemaReloadTest}. + * Configset-pinned-version behavior lives in {@link ConfigsetPinnedPackageVersionTest}. + * + *

Note on the class name: {@code BasePatternReplaceCharFilterFactory} and {@code + * BaseWhitespaceTokenizerFactory} (defined below) are referenced by fully-qualified name from + * pre-compiled test jars in {@code src/test-files/runtimecode/}. Renaming this class would break + * those binary fixtures. + */ @LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.ClusterPackage=DEBUG") public class TestPackages extends SolrCloudTestCase { @@ -549,115 +554,6 @@ private void verifyComponent( ":config:" + componentType + ":" + componentName + ":_packageinfo_:version", version)); } - @Test - @SuppressWarnings("unchecked") - public void testAPI() throws Exception { - String errPath = "/msg"; - String FILE1 = "/mypkg/v.0.12/jar_a.jar"; - String FILE2 = "/mypkg/v.0.12/jar_b.jar"; - String FILE3 = "/mypkg/v.0.13/jar_a.jar"; - - PackageApi.AddPackageVersion req = new PackageApi.AddPackageVersion("test_pkg"); - req.setVersion("0.12"); - req.setFiles(List.of(FILE1, FILE2)); - - // the files are not yet there. The command should fail with error saying "No such file" - expectError(req, cluster.getSolrClient(), errPath, "No such file:"); - - // post the jar file. No signature is sent - postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1, null); - - req.setFiles(List.of(FILE1)); - expectError(req, cluster.getSolrClient(), errPath, FILE1 + " has no signature"); - // now we upload the keys - byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); - // and upload the same file with a different name, but it has proper signature - postFileAndWait( - cluster, - "runtimecode/runtimelibs.jar.bin", - FILE2, - "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - // with correct signature - // after uploading the file, let's delete the keys to see if we get proper error message - req.setFiles(List.of(FILE2)); - /*expectError(req, cluster.getSolrClient(), errPath, - "ZooKeeper does not have any public keys");*/ - - // Now lets' put the keys back - - // this time we have a file with proper signature, public keys are in ZK - // so the add {} command should succeed - req.process(cluster.getSolrClient()); - - // Now verify the data in ZK - TestDistribFileStore.assertResponseValues( - 1, - () -> - NavigableObject.wrap( - Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH, null, new Stat()))), - Map.of(":packages:test_pkg[0]:version", "0.12", ":packages:test_pkg[0]:files[0]", FILE2)); - - // post a new jar with a proper signature - postFileAndWait( - cluster, - "runtimecode/runtimelibs_v2.jar.bin", - FILE3, - "j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA=="); - - // this time we are adding the second version of the package (0.13) - req.setVersion("0.13"); - req.setFiles(List.of(FILE3)); - - // this request should succeed - req.process(cluster.getSolrClient()); - // no verify the data (/packages.json) in ZK - TestDistribFileStore.assertResponseValues( - 1, - () -> - NavigableObject.wrap( - Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH, null, new Stat()))), - Map.of(":packages:test_pkg[1]:version", "0.13", ":packages:test_pkg[1]:files[0]", FILE3)); - - // Now we will just delete one version - PackageApi.DeletePackageVersion deleteReq = - new PackageApi.DeletePackageVersion("test_pkg", "0.1"); - - // we are expecting an error - expectError(deleteReq, cluster.getSolrClient(), errPath, "No such version:"); - - // correct version. Should succeed - new PackageApi.DeletePackageVersion("test_pkg", "0.12").process(cluster.getSolrClient()); - // Verify with ZK that the data is correct - TestDistribFileStore.assertResponseValues( - 1, - () -> - NavigableObject.wrap( - Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH, null, new Stat()))), - Map.of(":packages:test_pkg[0]:version", "0.13", ":packages:test_pkg[0]:files[0]", FILE3)); - - // So far we have been verifying the details with ZK directly - // use the package read API to verify with each node that it has the correct data - for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { - String path = jetty.getBaseURLV2().toString() + "/cluster/package?wt=javabin"; - TestDistribFileStore.assertResponseValues( - 10, - new Callable() { - @Override - public NavigableObject call() throws Exception { - HttpClient solrClient = jetty.getSolrClient().getHttpClient(); - byte[] bytes = solrClient.GET(path).getContent(); - return (NavigableObject) new JavaBinCodec().unmarshal(bytes); - } - }, - Map.of( - ":result:packages:test_pkg[0]:version", - "0.13", - ":result:packages:test_pkg[0]:files[0]", - FILE3)); - } - } - public static class C extends RequestHandlerBase implements SolrCoreAware { static boolean informCalled = false; @@ -695,121 +591,6 @@ public QParser createParser( } } - @Test - public void testSchemaPlugins() throws Exception { - String COLLECTION_NAME = "testSchemaLoadingColl"; - System.setProperty("managed.schema.mutable", "true"); - - IndexSchema[] schemas = new IndexSchema[2]; // tracks schemas for a selected core - - String FILE1 = "/schemapkg/schema-plugins.jar"; - byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); - postFileAndWait( - cluster, - "runtimecode/schema-plugins.jar.bin", - FILE1, - "U+AdO/jgY3DtMpeFRGoTQk72iA5g/qjPvdQYPGBaXB5+ggcTZk4FoIWiueB0bwGJ8Mg3V/elxOqEbD2JR8R0tA=="); - - String FILE2 = "/schemapkg/payload-component.jar"; - postFileAndWait( - cluster, - "runtimecode/payload-component.jar.bin", - FILE2, - "gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q=="); - - // upload package v1.0 - PackageApi.AddPackageVersion addReq = new PackageApi.AddPackageVersion("schemapkg"); - addReq.setVersion("1.0"); - addReq.setFiles(Arrays.asList(FILE1, FILE2)); - addReq.process(cluster.getSolrClient()); - - TestDistribFileStore.assertResponseValues( - 10, - () -> - new V2Request.Builder("/cluster/package") - .withMethod(SolrRequest.METHOD.GET) - .build() - .process(cluster.getSolrClient()), - Map.of( - ":result:packages:schemapkg[0]:version", - "1.0", - ":result:packages:schemapkg[0]:files[0]", - FILE1)); - - CollectionAdminRequest.createCollection(COLLECTION_NAME, "conf1", 2, 2) - .process(cluster.getSolrClient()); - cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); - - // make note of the schema instance for one of the cores - SolrCore.Provider coreProvider = - cluster.getJettySolrRunners().stream() - .flatMap( - jetty -> - jetty.getCoreContainer().getAllCoreNames().stream() - .map(name -> new SolrCore.Provider(jetty.getCoreContainer(), name, null))) - .findFirst() - .orElseThrow(); - - coreProvider.withCore(core -> schemas[0] = core.getLatestSchema()); - - // upload package v2.0 - addReq.setVersion("2.0"); - addReq.setFiles(Arrays.asList(FILE1, FILE2)); - addReq.process(cluster.getSolrClient()); - - TestDistribFileStore.assertResponseValues( - 10, - () -> - new V2Request.Builder("/cluster/package") - .withMethod(SolrRequest.METHOD.GET) - .build() - .process(cluster.getSolrClient()), - Map.of( - ":result:packages:schemapkg[1]:version", - "2.0", - ":result:packages:schemapkg[1]:files[0]", - FILE1)); - - // even though package version 2.0 uses exactly the same files - // as version 1.0, the core schema should still reload, and - // the core should be associated with a different schema instance - TestDistribFileStore.assertResponseValues( - 10, - () -> { - coreProvider.withCore(core -> schemas[1] = core.getLatestSchema()); - return params("schemaReloaded", (schemas[0] != schemas[1]) ? "yes" : "no"); - }, - Map.of("schemaReloaded", "yes")); - - // after the reload, the custom field type class now comes from package v2.0 - String fieldTypeName = "myNewTextFieldWithAnalyzerClass"; - - FieldType fieldTypeV1 = schemas[0].getFieldTypeByName(fieldTypeName); - assertEquals("my.pkg.MyTextField", fieldTypeV1.getClass().getCanonicalName()); - - FieldType fieldTypeV2 = schemas[1].getFieldTypeByName(fieldTypeName); - assertEquals("my.pkg.MyTextField", fieldTypeV2.getClass().getCanonicalName()); - - assertNotEquals( - "my.pkg.MyTextField classes should be from different classloaders", - fieldTypeV1.getClass(), - fieldTypeV2.getClass()); - } - - public static void postFileAndWait( - MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception { - ByteBuffer fileContent = getFileContent(fname); - @SuppressWarnings("ByteBufferBackingArray") // this is the result of a call to wrap() - String sha512 = DigestUtils.sha512Hex(fileContent.array()); - - TestDistribFileStore.postFile( - cluster.getSolrClient(), fileContent, path, sig); // has file, but no signature - - TestDistribFileStore.checkAllNodesForFile( - cluster, path, Map.of(":files:" + path + ":sha512", sha512), false); - } - private void expectError( SolrRequest req, SolrClient client, String errPath, String expectErrorMsg) throws IOException, SolrServerException {