From 7758c1ac18c00233f79c57399deca53504f2190e Mon Sep 17 00:00:00 2001 From: Prateek Singhal Date: Thu, 9 Oct 2025 11:02:32 -0700 Subject: [PATCH 1/5] SOLR-17949: Add Azure Blob Storage backup repository module This commit adds support for backing up and restoring Solr collections to Azure Blob Storage with multiple authentication options. Features: - Full backup/restore functionality to Azure Blob Storage - Support for 4 authentication methods: * Connection String (for development) * Account Name + Key (for simple production) * SAS Token (recommended for production) * Azure Identity (Managed Identity, Service Principal, Azure CLI) - Incremental backup support with versioning - Data integrity verification (checksum validation) - Compatible with Azurite emulator for local testing - Comprehensive documentation and 76 passing unit tests Implementation: - 8 implementation files (1,606 LOC) - 8 test files (2,180 LOC) - All dependencies Apache 2.0 licensed - Follows Solr's backup repository patterns --- gradle/libs.versions.toml | 8 + settings.gradle | 1 + solr/licenses/accessors-smart-2.5.0.jar.sha1 | 1 + solr/licenses/azure-LICENSE-ASL.txt | 206 +++++++ solr/licenses/azure-NOTICE.txt | 25 + solr/licenses/azure-core-1.52.0.jar.sha1 | 1 + .../azure-core-http-netty-1.15.4.jar.sha1 | 1 + solr/licenses/azure-identity-1.12.0.jar.sha1 | 1 + solr/licenses/azure-json-1.3.0.jar.sha1 | 1 + .../azure-storage-blob-12.25.0.jar.sha1 | 1 + .../azure-storage-common-12.25.0.jar.sha1 | 1 + ...ure-storage-internal-avro-12.10.0.jar.sha1 | 1 + solr/licenses/azure-xml-1.1.0.jar.sha1 | 1 + solr/licenses/content-type-2.3.jar.sha1 | 1 + solr/licenses/jna-platform-5.13.0.jar.sha1 | 1 + solr/licenses/json-smart-2.5.0.jar.sha1 | 1 + solr/licenses/msal4j-1.15.0.jar.sha1 | 1 + solr/licenses/msal4j-LICENSE-ASL.txt | 206 +++++++ solr/licenses/msal4j-NOTICE.txt | 25 + ...sal4j-persistence-extension-1.3.0.jar.sha1 | 1 + .../netty-buffer-4.1.110.Final.jar.sha1 | 1 + .../netty-codec-4.1.110.Final.jar.sha1 | 1 + .../netty-codec-dns-4.1.110.Final.jar.sha1 | 1 + .../netty-codec-http-4.1.110.Final.jar.sha1 | 1 + .../netty-codec-http2-4.1.110.Final.jar.sha1 | 1 + .../netty-codec-socks-4.1.110.Final.jar.sha1 | 1 + .../netty-common-4.1.110.Final.jar.sha1 | 1 + .../netty-handler-4.1.110.Final.jar.sha1 | 1 + ...netty-handler-proxy-4.1.110.Final.jar.sha1 | 1 + .../netty-resolver-4.1.110.Final.jar.sha1 | 1 + .../netty-resolver-dns-4.1.110.Final.jar.sha1 | 1 + ...r-dns-classes-macos-4.1.110.Final.jar.sha1 | 1 + ...ve-macos-4.1.110.Final-osx-x86_64.jar.sha1 | 1 + ...ive-boringssl-static-2.0.65.Final.jar.sha1 | 1 + ...tty-tcnative-classes-2.0.65.Final.jar.sha1 | 1 + .../netty-transport-4.1.110.Final.jar.sha1 | 1 + ...sport-classes-epoll-4.1.110.Final.jar.sha1 | 1 + ...port-classes-kqueue-4.1.110.Final.jar.sha1 | 1 + ...-epoll-4.1.110.Final-linux-x86_64.jar.sha1 | 1 + ...e-kqueue-4.1.110.Final-osx-x86_64.jar.sha1 | 1 + ...-native-unix-common-4.1.110.Final.jar.sha1 | 1 + solr/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 | 1 + solr/licenses/reactor-LICENSE-ASL.txt | 206 +++++++ solr/licenses/reactor-NOTICE.txt | 25 + solr/licenses/reactor-core-3.4.38.jar.sha1 | 1 + .../reactor-netty-core-1.0.45.jar.sha1 | 1 + .../reactor-netty-http-1.0.45.jar.sha1 | 1 + solr/modules/blob-repository/README.md | 473 +++++++++++++++ solr/modules/blob-repository/build.gradle | 51 ++ .../solr/blob/BlobBackupRepository.java | 407 +++++++++++++ .../solr/blob/BlobBackupRepositoryConfig.java | 80 +++ .../org/apache/solr/blob/BlobException.java | 31 + .../org/apache/solr/blob/BlobIndexInput.java | 199 +++++++ .../solr/blob/BlobNotFoundException.java | 24 + .../apache/solr/blob/BlobOutputStream.java | 280 +++++++++ .../apache/solr/blob/BlobStorageClient.java | 549 ++++++++++++++++++ .../org/apache/solr/blob/package-info.java | 44 ++ .../src/test-files/conf/schema.xml | 29 + .../src/test-files/conf/solrconfig.xml | 51 ++ .../blob-repository/src/test-files/log4j2.xml | 40 ++ .../solr/blob/AbstractBlobClientTest.java | 179 ++++++ .../solr/blob/BlobBackupRepositoryTest.java | 341 +++++++++++ .../solr/blob/BlobIncrementalBackupTest.java | 231 ++++++++ .../apache/solr/blob/BlobIndexInputTest.java | 287 +++++++++ .../solr/blob/BlobInstallShardTest.java | 276 +++++++++ .../solr/blob/BlobOutputStreamTest.java | 255 ++++++++ .../org/apache/solr/blob/BlobPathsTest.java | 332 +++++++++++ .../apache/solr/blob/BlobReadWriteTest.java | 281 +++++++++ .../pages/backup-restore.adoc | 252 +++++++- 69 files changed, 5432 insertions(+), 1 deletion(-) create mode 100644 solr/licenses/accessors-smart-2.5.0.jar.sha1 create mode 100644 solr/licenses/azure-LICENSE-ASL.txt create mode 100644 solr/licenses/azure-NOTICE.txt create mode 100644 solr/licenses/azure-core-1.52.0.jar.sha1 create mode 100644 solr/licenses/azure-core-http-netty-1.15.4.jar.sha1 create mode 100644 solr/licenses/azure-identity-1.12.0.jar.sha1 create mode 100644 solr/licenses/azure-json-1.3.0.jar.sha1 create mode 100644 solr/licenses/azure-storage-blob-12.25.0.jar.sha1 create mode 100644 solr/licenses/azure-storage-common-12.25.0.jar.sha1 create mode 100644 solr/licenses/azure-storage-internal-avro-12.10.0.jar.sha1 create mode 100644 solr/licenses/azure-xml-1.1.0.jar.sha1 create mode 100644 solr/licenses/content-type-2.3.jar.sha1 create mode 100644 solr/licenses/jna-platform-5.13.0.jar.sha1 create mode 100644 solr/licenses/json-smart-2.5.0.jar.sha1 create mode 100644 solr/licenses/msal4j-1.15.0.jar.sha1 create mode 100644 solr/licenses/msal4j-LICENSE-ASL.txt create mode 100644 solr/licenses/msal4j-NOTICE.txt create mode 100644 solr/licenses/msal4j-persistence-extension-1.3.0.jar.sha1 create mode 100644 solr/licenses/netty-buffer-4.1.110.Final.jar.sha1 create mode 100644 solr/licenses/netty-codec-4.1.110.Final.jar.sha1 create mode 100644 solr/licenses/netty-codec-dns-4.1.110.Final.jar.sha1 create mode 100644 solr/licenses/netty-codec-http-4.1.110.Final.jar.sha1 create mode 100644 solr/licenses/netty-codec-http2-4.1.110.Final.jar.sha1 create mode 100644 solr/licenses/netty-codec-socks-4.1.110.Final.jar.sha1 create mode 100644 solr/licenses/netty-common-4.1.110.Final.jar.sha1 create mode 100644 solr/licenses/netty-handler-4.1.110.Final.jar.sha1 create mode 100644 solr/licenses/netty-handler-proxy-4.1.110.Final.jar.sha1 create mode 100644 solr/licenses/netty-resolver-4.1.110.Final.jar.sha1 create mode 100644 solr/licenses/netty-resolver-dns-4.1.110.Final.jar.sha1 create mode 100644 solr/licenses/netty-resolver-dns-classes-macos-4.1.110.Final.jar.sha1 create mode 100644 solr/licenses/netty-resolver-dns-native-macos-4.1.110.Final-osx-x86_64.jar.sha1 create mode 100644 solr/licenses/netty-tcnative-boringssl-static-2.0.65.Final.jar.sha1 create mode 100644 solr/licenses/netty-tcnative-classes-2.0.65.Final.jar.sha1 create mode 100644 solr/licenses/netty-transport-4.1.110.Final.jar.sha1 create mode 100644 solr/licenses/netty-transport-classes-epoll-4.1.110.Final.jar.sha1 create mode 100644 solr/licenses/netty-transport-classes-kqueue-4.1.110.Final.jar.sha1 create mode 100644 solr/licenses/netty-transport-native-epoll-4.1.110.Final-linux-x86_64.jar.sha1 create mode 100644 solr/licenses/netty-transport-native-kqueue-4.1.110.Final-osx-x86_64.jar.sha1 create mode 100644 solr/licenses/netty-transport-native-unix-common-4.1.110.Final.jar.sha1 create mode 100644 solr/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 create mode 100644 solr/licenses/reactor-LICENSE-ASL.txt create mode 100644 solr/licenses/reactor-NOTICE.txt create mode 100644 solr/licenses/reactor-core-3.4.38.jar.sha1 create mode 100644 solr/licenses/reactor-netty-core-1.0.45.jar.sha1 create mode 100644 solr/licenses/reactor-netty-http-1.0.45.jar.sha1 create mode 100644 solr/modules/blob-repository/README.md create mode 100644 solr/modules/blob-repository/build.gradle create mode 100644 solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobBackupRepository.java create mode 100644 solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobBackupRepositoryConfig.java create mode 100644 solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobException.java create mode 100644 solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobIndexInput.java create mode 100644 solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobNotFoundException.java create mode 100644 solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobOutputStream.java create mode 100644 solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobStorageClient.java create mode 100644 solr/modules/blob-repository/src/java/org/apache/solr/blob/package-info.java create mode 100644 solr/modules/blob-repository/src/test-files/conf/schema.xml create mode 100644 solr/modules/blob-repository/src/test-files/conf/solrconfig.xml create mode 100644 solr/modules/blob-repository/src/test-files/log4j2.xml create mode 100644 solr/modules/blob-repository/src/test/org/apache/solr/blob/AbstractBlobClientTest.java create mode 100644 solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobBackupRepositoryTest.java create mode 100644 solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobIncrementalBackupTest.java create mode 100644 solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobIndexInputTest.java create mode 100644 solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobInstallShardTest.java create mode 100644 solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobOutputStreamTest.java create mode 100644 solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobPathsTest.java create mode 100644 solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobReadWriteTest.java diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 634491432d9a..c63808a26071 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -50,6 +50,10 @@ aqute-bnd = "6.4.1" asciidoctor-mathjax = "0.0.9" # @keep Asciidoctor tabs version used in ref-guide asciidoctor-tabs = "1.0.0-beta.6" +azure-storage = "12.25.0" +azure-identity = "1.12.0" +azure-core = "1.52.0" +azure-core-http-netty = "1.15.4" # @keep bats-assert (node) version used in packaging bats-assert = "2.0.0" # @keep bats-core (node) version used in packaging @@ -304,6 +308,10 @@ apache-zookeeper-zookeeper = { module = "org.apache.zookeeper:zookeeper", versio # @keep transitive dependency for version alignment apiguardian-api = { module = "org.apiguardian:apiguardian-api", version.ref = "apiguardian" } aqute-bnd-annotation = { module = "biz.aQute.bnd:biz.aQute.bnd.annotation", version.ref = "aqute-bnd" } +azure-storage-blob = { module = "com.azure:azure-storage-blob", version.ref = "azure-storage" } +azure-identity = { module = "com.azure:azure-identity", version.ref = "azure-identity" } +azure-core = { module = "com.azure:azure-core", version.ref = "azure-core" } +azure-core-http-netty = { module = "com.azure:azure-core-http-netty", version.ref = "azure-core-http-netty" } bc-jose4j = { module = "org.bitbucket.b_c:jose4j", version.ref = "bc-jose4j" } benmanes-caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version.ref = "benmanes-caffeine" } bouncycastle-bcpkix = { module = "org.bouncycastle:bcpkix-jdk18on", version.ref = "bouncycastle" } diff --git a/settings.gradle b/settings.gradle index 7b635cbbeb9d..eff852fb9c65 100644 --- a/settings.gradle +++ b/settings.gradle @@ -44,6 +44,7 @@ include "solr:core" include "solr:cross-dc-manager" include "solr:server" include "solr:modules:analysis-extras" +include "solr:modules:blob-repository" include "solr:modules:clustering" include "solr:modules:cross-dc" include "solr:modules:cuvs" diff --git a/solr/licenses/accessors-smart-2.5.0.jar.sha1 b/solr/licenses/accessors-smart-2.5.0.jar.sha1 new file mode 100644 index 000000000000..60d26d2d99fa --- /dev/null +++ b/solr/licenses/accessors-smart-2.5.0.jar.sha1 @@ -0,0 +1 @@ +aca011492dfe9c26f4e0659028a4fe0970829dd8 diff --git a/solr/licenses/azure-LICENSE-ASL.txt b/solr/licenses/azure-LICENSE-ASL.txt new file mode 100644 index 000000000000..1eef70a9b9f4 --- /dev/null +++ b/solr/licenses/azure-LICENSE-ASL.txt @@ -0,0 +1,206 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Note: Other license terms may apply to certain, identified software files contained within or distributed + with the accompanying software if such terms are included in the directory containing the accompanying software. + Such other license terms will then apply in lieu of the terms of the software license above. diff --git a/solr/licenses/azure-NOTICE.txt b/solr/licenses/azure-NOTICE.txt new file mode 100644 index 000000000000..7b5a06890325 --- /dev/null +++ b/solr/licenses/azure-NOTICE.txt @@ -0,0 +1,25 @@ +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). diff --git a/solr/licenses/azure-core-1.52.0.jar.sha1 b/solr/licenses/azure-core-1.52.0.jar.sha1 new file mode 100644 index 000000000000..e0d4f012e79d --- /dev/null +++ b/solr/licenses/azure-core-1.52.0.jar.sha1 @@ -0,0 +1 @@ +43bd4ad76e6772d24c545635b48e0ed4d0e511f2 diff --git a/solr/licenses/azure-core-http-netty-1.15.4.jar.sha1 b/solr/licenses/azure-core-http-netty-1.15.4.jar.sha1 new file mode 100644 index 000000000000..614ec2b5b116 --- /dev/null +++ b/solr/licenses/azure-core-http-netty-1.15.4.jar.sha1 @@ -0,0 +1 @@ +489a38c9e6efb5ce01fbd276d8cb6c0e89000459 diff --git a/solr/licenses/azure-identity-1.12.0.jar.sha1 b/solr/licenses/azure-identity-1.12.0.jar.sha1 new file mode 100644 index 000000000000..1dcd782fa8d0 --- /dev/null +++ b/solr/licenses/azure-identity-1.12.0.jar.sha1 @@ -0,0 +1 @@ +1d7efb089db2fe7a60526b8ff50b0c681fe1b079 diff --git a/solr/licenses/azure-json-1.3.0.jar.sha1 b/solr/licenses/azure-json-1.3.0.jar.sha1 new file mode 100644 index 000000000000..47daa904564b --- /dev/null +++ b/solr/licenses/azure-json-1.3.0.jar.sha1 @@ -0,0 +1 @@ +11b6a0708e9d6c90a1a76574c7720edce47dacc1 diff --git a/solr/licenses/azure-storage-blob-12.25.0.jar.sha1 b/solr/licenses/azure-storage-blob-12.25.0.jar.sha1 new file mode 100644 index 000000000000..1cfc20dfc28d --- /dev/null +++ b/solr/licenses/azure-storage-blob-12.25.0.jar.sha1 @@ -0,0 +1 @@ +94e0aed4a4cc8496d813e4432f840cb284b47ac5 diff --git a/solr/licenses/azure-storage-common-12.25.0.jar.sha1 b/solr/licenses/azure-storage-common-12.25.0.jar.sha1 new file mode 100644 index 000000000000..6aacac9e105e --- /dev/null +++ b/solr/licenses/azure-storage-common-12.25.0.jar.sha1 @@ -0,0 +1 @@ +4c2c2eebb4195fa186a26257572789dd31f86493 diff --git a/solr/licenses/azure-storage-internal-avro-12.10.0.jar.sha1 b/solr/licenses/azure-storage-internal-avro-12.10.0.jar.sha1 new file mode 100644 index 000000000000..3446b7706813 --- /dev/null +++ b/solr/licenses/azure-storage-internal-avro-12.10.0.jar.sha1 @@ -0,0 +1 @@ +8fe0d236b37610be22944a69332f79e880b7203f diff --git a/solr/licenses/azure-xml-1.1.0.jar.sha1 b/solr/licenses/azure-xml-1.1.0.jar.sha1 new file mode 100644 index 000000000000..1224ee5783bb --- /dev/null +++ b/solr/licenses/azure-xml-1.1.0.jar.sha1 @@ -0,0 +1 @@ +8218a00c07f9f66d5dc7ae2ba613da6890867497 diff --git a/solr/licenses/content-type-2.3.jar.sha1 b/solr/licenses/content-type-2.3.jar.sha1 new file mode 100644 index 000000000000..7718175e95f9 --- /dev/null +++ b/solr/licenses/content-type-2.3.jar.sha1 @@ -0,0 +1 @@ +e3aa0be212d7a42839a8f3f506f5b990bcce0222 diff --git a/solr/licenses/jna-platform-5.13.0.jar.sha1 b/solr/licenses/jna-platform-5.13.0.jar.sha1 new file mode 100644 index 000000000000..2c60ada13780 --- /dev/null +++ b/solr/licenses/jna-platform-5.13.0.jar.sha1 @@ -0,0 +1 @@ +88e9a306715e9379f3122415ef4ae759a352640d diff --git a/solr/licenses/json-smart-2.5.0.jar.sha1 b/solr/licenses/json-smart-2.5.0.jar.sha1 new file mode 100644 index 000000000000..2c839a3e5af1 --- /dev/null +++ b/solr/licenses/json-smart-2.5.0.jar.sha1 @@ -0,0 +1 @@ +57a64f421b472849c40e77d2e7cce3a141b41e99 diff --git a/solr/licenses/msal4j-1.15.0.jar.sha1 b/solr/licenses/msal4j-1.15.0.jar.sha1 new file mode 100644 index 000000000000..25d68664fd0b --- /dev/null +++ b/solr/licenses/msal4j-1.15.0.jar.sha1 @@ -0,0 +1 @@ +52fd60d5dc3f0fb3ed5c19b63f6f2312cd1f6add diff --git a/solr/licenses/msal4j-LICENSE-ASL.txt b/solr/licenses/msal4j-LICENSE-ASL.txt new file mode 100644 index 000000000000..1eef70a9b9f4 --- /dev/null +++ b/solr/licenses/msal4j-LICENSE-ASL.txt @@ -0,0 +1,206 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Note: Other license terms may apply to certain, identified software files contained within or distributed + with the accompanying software if such terms are included in the directory containing the accompanying software. + Such other license terms will then apply in lieu of the terms of the software license above. diff --git a/solr/licenses/msal4j-NOTICE.txt b/solr/licenses/msal4j-NOTICE.txt new file mode 100644 index 000000000000..7b5a06890325 --- /dev/null +++ b/solr/licenses/msal4j-NOTICE.txt @@ -0,0 +1,25 @@ +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). diff --git a/solr/licenses/msal4j-persistence-extension-1.3.0.jar.sha1 b/solr/licenses/msal4j-persistence-extension-1.3.0.jar.sha1 new file mode 100644 index 000000000000..0131bb7b2a04 --- /dev/null +++ b/solr/licenses/msal4j-persistence-extension-1.3.0.jar.sha1 @@ -0,0 +1 @@ +8a8ef1517d27a5b4de1512ef94679bdb59f210b6 diff --git a/solr/licenses/netty-buffer-4.1.110.Final.jar.sha1 b/solr/licenses/netty-buffer-4.1.110.Final.jar.sha1 new file mode 100644 index 000000000000..bb8c75abbcdf --- /dev/null +++ b/solr/licenses/netty-buffer-4.1.110.Final.jar.sha1 @@ -0,0 +1 @@ +3d918a9ee057d995c362902b54634fc307132aac diff --git a/solr/licenses/netty-codec-4.1.110.Final.jar.sha1 b/solr/licenses/netty-codec-4.1.110.Final.jar.sha1 new file mode 100644 index 000000000000..a41772233da8 --- /dev/null +++ b/solr/licenses/netty-codec-4.1.110.Final.jar.sha1 @@ -0,0 +1 @@ +f1fa43b03e93ab88e805b6a4e3e83780c80b47d2 diff --git a/solr/licenses/netty-codec-dns-4.1.110.Final.jar.sha1 b/solr/licenses/netty-codec-dns-4.1.110.Final.jar.sha1 new file mode 100644 index 000000000000..0cb6e0d23a43 --- /dev/null +++ b/solr/licenses/netty-codec-dns-4.1.110.Final.jar.sha1 @@ -0,0 +1 @@ +381c5bf8b7570c163fa7893a26d02b7ac36ff6eb diff --git a/solr/licenses/netty-codec-http-4.1.110.Final.jar.sha1 b/solr/licenses/netty-codec-http-4.1.110.Final.jar.sha1 new file mode 100644 index 000000000000..00574566267e --- /dev/null +++ b/solr/licenses/netty-codec-http-4.1.110.Final.jar.sha1 @@ -0,0 +1 @@ +9d05cd927209ea25bbf342962c00b8e5a828c2a4 diff --git a/solr/licenses/netty-codec-http2-4.1.110.Final.jar.sha1 b/solr/licenses/netty-codec-http2-4.1.110.Final.jar.sha1 new file mode 100644 index 000000000000..27bcd9e7dc43 --- /dev/null +++ b/solr/licenses/netty-codec-http2-4.1.110.Final.jar.sha1 @@ -0,0 +1 @@ +e0849843eb5b1c036b12551baca98a9f7ff847a0 diff --git a/solr/licenses/netty-codec-socks-4.1.110.Final.jar.sha1 b/solr/licenses/netty-codec-socks-4.1.110.Final.jar.sha1 new file mode 100644 index 000000000000..0c7f8c8d5411 --- /dev/null +++ b/solr/licenses/netty-codec-socks-4.1.110.Final.jar.sha1 @@ -0,0 +1 @@ +4d54c8d5b95b14756043efb59b8c3e62ec67aa43 diff --git a/solr/licenses/netty-common-4.1.110.Final.jar.sha1 b/solr/licenses/netty-common-4.1.110.Final.jar.sha1 new file mode 100644 index 000000000000..588f41bee630 --- /dev/null +++ b/solr/licenses/netty-common-4.1.110.Final.jar.sha1 @@ -0,0 +1 @@ +ec361e7e025c029be50c55c8480080cabcbc01e7 diff --git a/solr/licenses/netty-handler-4.1.110.Final.jar.sha1 b/solr/licenses/netty-handler-4.1.110.Final.jar.sha1 new file mode 100644 index 000000000000..8946e71e1483 --- /dev/null +++ b/solr/licenses/netty-handler-4.1.110.Final.jar.sha1 @@ -0,0 +1 @@ +168db749c22652ee7fed1ebf7ec46ce856d75e51 diff --git a/solr/licenses/netty-handler-proxy-4.1.110.Final.jar.sha1 b/solr/licenses/netty-handler-proxy-4.1.110.Final.jar.sha1 new file mode 100644 index 000000000000..33ded80b73e3 --- /dev/null +++ b/solr/licenses/netty-handler-proxy-4.1.110.Final.jar.sha1 @@ -0,0 +1 @@ +b7fb401dd47c79e6b99f2319ac3b561c50c31c30 diff --git a/solr/licenses/netty-resolver-4.1.110.Final.jar.sha1 b/solr/licenses/netty-resolver-4.1.110.Final.jar.sha1 new file mode 100644 index 000000000000..50c7d92a43e8 --- /dev/null +++ b/solr/licenses/netty-resolver-4.1.110.Final.jar.sha1 @@ -0,0 +1 @@ +66c15921104cda0159b34e316541bc765dfaf3c0 diff --git a/solr/licenses/netty-resolver-dns-4.1.110.Final.jar.sha1 b/solr/licenses/netty-resolver-dns-4.1.110.Final.jar.sha1 new file mode 100644 index 000000000000..1eb243870cf1 --- /dev/null +++ b/solr/licenses/netty-resolver-dns-4.1.110.Final.jar.sha1 @@ -0,0 +1 @@ +3e687cdc4ecdbbad07508a11b715bdf95fa20939 diff --git a/solr/licenses/netty-resolver-dns-classes-macos-4.1.110.Final.jar.sha1 b/solr/licenses/netty-resolver-dns-classes-macos-4.1.110.Final.jar.sha1 new file mode 100644 index 000000000000..2be06f13e0c7 --- /dev/null +++ b/solr/licenses/netty-resolver-dns-classes-macos-4.1.110.Final.jar.sha1 @@ -0,0 +1 @@ +4be9633daf46657dd94851ce44adaea14a2faa7e diff --git a/solr/licenses/netty-resolver-dns-native-macos-4.1.110.Final-osx-x86_64.jar.sha1 b/solr/licenses/netty-resolver-dns-native-macos-4.1.110.Final-osx-x86_64.jar.sha1 new file mode 100644 index 000000000000..63f71cb28d3e --- /dev/null +++ b/solr/licenses/netty-resolver-dns-native-macos-4.1.110.Final-osx-x86_64.jar.sha1 @@ -0,0 +1 @@ +6376510bb8a8c755a1f0af1d27c2902a1c84f58c diff --git a/solr/licenses/netty-tcnative-boringssl-static-2.0.65.Final.jar.sha1 b/solr/licenses/netty-tcnative-boringssl-static-2.0.65.Final.jar.sha1 new file mode 100644 index 000000000000..c083dbe75686 --- /dev/null +++ b/solr/licenses/netty-tcnative-boringssl-static-2.0.65.Final.jar.sha1 @@ -0,0 +1 @@ +b31c6944d9cfd596b6c25fe17e36780bfa2d7473 diff --git a/solr/licenses/netty-tcnative-classes-2.0.65.Final.jar.sha1 b/solr/licenses/netty-tcnative-classes-2.0.65.Final.jar.sha1 new file mode 100644 index 000000000000..f95844b2b89f --- /dev/null +++ b/solr/licenses/netty-tcnative-classes-2.0.65.Final.jar.sha1 @@ -0,0 +1 @@ +3a7aecd4bcaf75c7b0b02c26ea6ceacf3e8f5f4d diff --git a/solr/licenses/netty-transport-4.1.110.Final.jar.sha1 b/solr/licenses/netty-transport-4.1.110.Final.jar.sha1 new file mode 100644 index 000000000000..29293a1ab6d5 --- /dev/null +++ b/solr/licenses/netty-transport-4.1.110.Final.jar.sha1 @@ -0,0 +1 @@ +b91f04c39ac14d6a29d07184ef305953ee6e0348 diff --git a/solr/licenses/netty-transport-classes-epoll-4.1.110.Final.jar.sha1 b/solr/licenses/netty-transport-classes-epoll-4.1.110.Final.jar.sha1 new file mode 100644 index 000000000000..75620db21e76 --- /dev/null +++ b/solr/licenses/netty-transport-classes-epoll-4.1.110.Final.jar.sha1 @@ -0,0 +1 @@ +3ca1cff0bf82bfd38e89f6946e54f24cbb3424a2 diff --git a/solr/licenses/netty-transport-classes-kqueue-4.1.110.Final.jar.sha1 b/solr/licenses/netty-transport-classes-kqueue-4.1.110.Final.jar.sha1 new file mode 100644 index 000000000000..db1bf439ad40 --- /dev/null +++ b/solr/licenses/netty-transport-classes-kqueue-4.1.110.Final.jar.sha1 @@ -0,0 +1 @@ +ae6037a535779ba61e316551cc6245eb1707ff7a diff --git a/solr/licenses/netty-transport-native-epoll-4.1.110.Final-linux-x86_64.jar.sha1 b/solr/licenses/netty-transport-native-epoll-4.1.110.Final-linux-x86_64.jar.sha1 new file mode 100644 index 000000000000..5d194b1e7dbf --- /dev/null +++ b/solr/licenses/netty-transport-native-epoll-4.1.110.Final-linux-x86_64.jar.sha1 @@ -0,0 +1 @@ +72b74a82d22e215d1f2573c040078e0afff519af diff --git a/solr/licenses/netty-transport-native-kqueue-4.1.110.Final-osx-x86_64.jar.sha1 b/solr/licenses/netty-transport-native-kqueue-4.1.110.Final-osx-x86_64.jar.sha1 new file mode 100644 index 000000000000..9821c7805fc0 --- /dev/null +++ b/solr/licenses/netty-transport-native-kqueue-4.1.110.Final-osx-x86_64.jar.sha1 @@ -0,0 +1 @@ +d153b25a358851f15acdd70aeb43e6830500a6be diff --git a/solr/licenses/netty-transport-native-unix-common-4.1.110.Final.jar.sha1 b/solr/licenses/netty-transport-native-unix-common-4.1.110.Final.jar.sha1 new file mode 100644 index 000000000000..8e0a7bd52bc9 --- /dev/null +++ b/solr/licenses/netty-transport-native-unix-common-4.1.110.Final.jar.sha1 @@ -0,0 +1 @@ +a7096e7c0a25a983647909d7513f5d4943d589c0 diff --git a/solr/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 b/solr/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 new file mode 100644 index 000000000000..3d7d85862600 --- /dev/null +++ b/solr/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 @@ -0,0 +1 @@ +fa9a2e447e2cef4dfda40a854dd7ec35624a7799 diff --git a/solr/licenses/reactor-LICENSE-ASL.txt b/solr/licenses/reactor-LICENSE-ASL.txt new file mode 100644 index 000000000000..1eef70a9b9f4 --- /dev/null +++ b/solr/licenses/reactor-LICENSE-ASL.txt @@ -0,0 +1,206 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Note: Other license terms may apply to certain, identified software files contained within or distributed + with the accompanying software if such terms are included in the directory containing the accompanying software. + Such other license terms will then apply in lieu of the terms of the software license above. diff --git a/solr/licenses/reactor-NOTICE.txt b/solr/licenses/reactor-NOTICE.txt new file mode 100644 index 000000000000..7b5a06890325 --- /dev/null +++ b/solr/licenses/reactor-NOTICE.txt @@ -0,0 +1,25 @@ +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). diff --git a/solr/licenses/reactor-core-3.4.38.jar.sha1 b/solr/licenses/reactor-core-3.4.38.jar.sha1 new file mode 100644 index 000000000000..1ca673ac48c5 --- /dev/null +++ b/solr/licenses/reactor-core-3.4.38.jar.sha1 @@ -0,0 +1 @@ +94178266e36e6de6338a1c180efaddcff0251002 diff --git a/solr/licenses/reactor-netty-core-1.0.45.jar.sha1 b/solr/licenses/reactor-netty-core-1.0.45.jar.sha1 new file mode 100644 index 000000000000..e241697b42e3 --- /dev/null +++ b/solr/licenses/reactor-netty-core-1.0.45.jar.sha1 @@ -0,0 +1 @@ +42aea422b0551b1db4dd4eddf598ccddd5408a4e diff --git a/solr/licenses/reactor-netty-http-1.0.45.jar.sha1 b/solr/licenses/reactor-netty-http-1.0.45.jar.sha1 new file mode 100644 index 000000000000..061f41d113ae --- /dev/null +++ b/solr/licenses/reactor-netty-http-1.0.45.jar.sha1 @@ -0,0 +1 @@ +f24886830010329239a2f10f19727ea420898fba diff --git a/solr/modules/blob-repository/README.md b/solr/modules/blob-repository/README.md new file mode 100644 index 000000000000..3deab497a674 --- /dev/null +++ b/solr/modules/blob-repository/README.md @@ -0,0 +1,473 @@ + + +Apache Solr - Azure Blob Storage Repository +=========================================== + +This Azure Blob Storage repository is a backup repository implementation designed to provide backup/restore functionality to Azure Blob Storage. + +## Quick Start + +**Choose your authentication method:** + +- 🚀 **Local Development?** → Use **Connection String** (simplest) +- 🔐 **Production on Azure VM/AKS?** → Use **Managed Identity** (most secure) +- 🏢 **Production elsewhere?** → Use **Service Principal** or **SAS Token** +- 🧪 **Testing?** → Use **Azure CLI** (no config changes) + +**Prerequisites:** +- Azure Storage Account with a blob container +- Container must already exist (e.g., `solr-backup`) +- Solr blob-repository module enabled +- Network access to Azure Blob Storage (HTTPS port 443) + +## Prerequisites + +Before configuring authentication, ensure you have: + +1. **Azure Storage Account** - Created and accessible +2. **Blob Container** - Must already exist in your storage account + ```bash + # Create container using Azure CLI + az storage container create \ + --name solr-backup \ + --account-name YOUR_ACCOUNT_NAME + ``` +3. **Solr Module** - Enable blob-repository module: + ```bash + export SOLR_MODULES=blob-repository + ./bin/solr start + ``` +4. **Network Access** - Solr can reach Azure Blob Storage (HTTPS port 443) + +Optional (depending on authentication method): +- **Azure CLI** installed and configured (`az login`) +- **RBAC Permissions** for Azure Identity methods +- **SAS Token** or **Account Keys** from Azure Portal + +## Authentication Options + +The Azure Blob Storage backup repository supports four authentication methods. Choose the one that best fits your security requirements and deployment environment. + +### 1. Connection String + +The simplest authentication method using a full connection string. + +#### Configuration in solr.xml: +```xml + + + YOUR_CONTAINER_NAME + DefaultEndpointsProtocol=https;AccountName=YOUR_ACCOUNT_NAME;AccountKey=YOUR_ACCOUNT_KEY;EndpointSuffix=core.windows.net + + +``` + +**Note:** This method is simple but exposes the account key in configuration. Not recommended for production environments. + +### 2. Account Name + Key + +Separates the account credentials from the endpoint configuration. + +#### Configuration in solr.xml: +```xml + + + YOUR_CONTAINER_NAME + https://YOUR_ACCOUNT_NAME.blob.core.windows.net + YOUR_ACCOUNT_NAME + YOUR_ACCOUNT_KEY + + +``` + +**Note:** Similar to connection string, this exposes the account key. Use with caution in production. + +### 3. SAS Token (Recommended for Production) + +**Important:** The SAS token must be configured with proper permissions to work correctly. + +#### Required SAS Token Configuration: +- **Allowed services:** Blob +- **Allowed resource types:** Service, Container, Object (`srt=sco`) +- **Allowed permissions:** Read, Write, Delete, List, Add, Create (`sp=rwdlac` minimum) +- **Protocol:** HTTPS only +- **Expiry:** Set appropriate expiration time (e.g., 1 year) + +#### Generating SAS Token (Azure Portal): +1. Navigate to your Storage Account +2. Click "Shared access signature" (left menu under "Security + networking") +3. Configure: + - Allowed services: ☑ Blob + - Allowed resource types: ☑ Service, ☑ Container, ☑ Object + - Allowed permissions: ☑ Read, ☑ Write, ☑ Delete, ☑ List, ☑ Add, ☑ Create + - Start/Expiry time: Set your desired validity period + - Allowed protocols: HTTPS only +4. Click "Generate SAS and connection string" +5. Copy the **SAS token** (remove the leading `?` if present) + +#### Generating SAS Token (Azure CLI): +```bash +az storage account generate-sas \ + --account-name YOUR_ACCOUNT_NAME \ + --services b \ + --resource-types sco \ + --permissions rwdlac \ + --expiry 2026-12-31T23:59:59Z \ + --https-only \ + --output tsv +``` + +#### Configuration in solr.xml: +```xml + + + YOUR_CONTAINER_NAME + https://YOUR_ACCOUNT_NAME.blob.core.windows.net + sv=2024-11-04&ss=b&srt=sco&sp=rwdlac&se=2026-12-31T23:59:59Z&st=2025-01-01T00:00:00Z&spr=https&sig=YOUR_SIGNATURE + + +``` + +**Note:** In XML, `&` characters in the SAS token must be escaped as `&`. The container must already exist in Azure Blob Storage before using it with Solr. + +#### Why SAS Token? +- ✅ Time-limited access (automatically expires) +- ✅ Scoped permissions (can restrict to specific operations) +- ✅ Revocable without rotating account keys +- ✅ No account key exposure in configuration +- ✅ Can restrict to specific IP addresses + +### 4. Azure Identity (Best for Production) + +Uses Azure Active Directory (Entra ID) for authentication. Provides enterprise-grade security with **no credentials in configuration files**. + +Azure Identity supports three authentication methods: +- **Azure CLI** - For local development +- **Service Principal** - For automation and CI/CD +- **Managed Identity** - For Azure VMs/AKS (no credentials needed) + +--- + +#### Option A: Azure CLI (Local Development) + +Best for local development and testing. Uses your Azure login credentials. + +**Prerequisites:** +- Azure CLI installed and logged in (`az login`) +- User account has "Storage Blob Data Contributor" role + +**Grant permissions:** +```bash +# Get your user's Object ID +USER_OBJECT_ID=$(az ad signed-in-user show --query id -o tsv) + +# Grant Storage Blob Data Contributor role +az role assignment create \ + --role "Storage Blob Data Contributor" \ + --assignee $USER_OBJECT_ID \ + --scope /subscriptions/YOUR_SUBSCRIPTION_ID/resourceGroups/YOUR_RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts/YOUR_ACCOUNT_NAME +``` + +**Configuration in solr.xml:** +```xml + + + YOUR_CONTAINER_NAME + https://YOUR_ACCOUNT_NAME.blob.core.windows.net + + + +``` + +--- + +#### Option B: Service Principal (Automation/CI-CD) + +Best for automation, CI/CD pipelines, and production deployments outside of Azure. + +**Create Service Principal:** +```bash +az ad sp create-for-rbac \ + --name "solr-backup-sp" \ + --role "Storage Blob Data Contributor" \ + --scopes /subscriptions/YOUR_SUBSCRIPTION_ID/resourceGroups/YOUR_RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts/YOUR_ACCOUNT_NAME + +# Output: +# { +# "appId": "CLIENT_ID", +# "password": "CLIENT_SECRET", +# "tenant": "TENANT_ID" +# } +``` + +**Configuration in solr.xml:** +```xml + + + YOUR_CONTAINER_NAME + https://YOUR_ACCOUNT_NAME.blob.core.windows.net + YOUR_TENANT_ID + YOUR_CLIENT_ID + YOUR_CLIENT_SECRET + + +``` + +**Alternative: Environment Variables** + +Instead of putting credentials in solr.xml, you can use environment variables: +```bash +export AZURE_TENANT_ID="your-tenant-id" +export AZURE_CLIENT_ID="your-client-id" +export AZURE_CLIENT_SECRET="your-client-secret" +``` + +Then solr.xml only needs: +```xml + + + YOUR_CONTAINER_NAME + https://YOUR_ACCOUNT_NAME.blob.core.windows.net + + + +``` + +--- + +#### Option C: Managed Identity (Azure VM/AKS) + +Best for production workloads running on Azure infrastructure. **Most secure** - no credentials at all! + +**Enable Managed Identity:** +```bash +# For Azure VM +az vm identity assign \ + --name YOUR_VM_NAME \ + --resource-group YOUR_RESOURCE_GROUP + +# Get the managed identity principal ID +PRINCIPAL_ID=$(az vm identity show \ + --name YOUR_VM_NAME \ + --resource-group YOUR_RESOURCE_GROUP \ + --query principalId -o tsv) + +# Grant Storage Blob Data Contributor role +az role assignment create \ + --role "Storage Blob Data Contributor" \ + --assignee $PRINCIPAL_ID \ + --scope /subscriptions/YOUR_SUBSCRIPTION_ID/resourceGroups/YOUR_RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts/YOUR_ACCOUNT_NAME +``` + +**Configuration in solr.xml:** +```xml + + + YOUR_CONTAINER_NAME + https://YOUR_ACCOUNT_NAME.blob.core.windows.net + + + +``` + +--- + +#### Why Use Azure Identity? + +**Security Benefits:** +- ✅ **Zero secrets** in configuration files +- ✅ **Automatic credential rotation** via Azure AD +- ✅ **Fine-grained RBAC** access control +- ✅ **Full audit logging** via Azure AD +- ✅ **Compliance-friendly** (SOC 2, ISO 27001, etc.) +- ✅ **Token-based** authentication (short-lived tokens) + +**Operational Benefits:** +- ✅ **No credential management** overhead +- ✅ **Works across environments** (dev, staging, prod) +- ✅ **Integrates with Azure services** seamlessly +- ✅ **Supports multiple identities** (users, service principals, managed identities) + +**Performance:** +- Slightly slower than key-based auth (~5-10 seconds overhead for token acquisition) +- Negligible for large backups (overhead is constant, not proportional to data size) +- Well worth the security benefits + +## Authentication Comparison + +| Method | Security | Setup | Best For | Credentials in Config | Production | +|--------|----------|-------|----------|----------------------|------------| +| Connection String | ⚠️ Low | ⭐ Simple | Development | ❌ Full account key | ❌ Dev only | +| Account Key | ⚠️ Low | ⭐ Simple | Development | ❌ Full account key | ⚠️ Caution | +| **SAS Token** | ✅ Good | ⭐⭐ Medium | **Production** | ⚠️ Time-limited token | ✅ **Recommended** | +| Azure Identity (CLI) | ✅ Excellent | ⭐⭐ Medium | Local Dev/Test | ✅ None (uses login) | ✅ Dev/Test | +| **Azure Identity (SP)** | ✅ Excellent | ⭐⭐⭐ Complex | **CI/CD/Production** | ⚠️ Scoped credentials | ✅ **Recommended** | +| Azure Identity (MI) | ✅✅ Best | ⭐⭐⭐ Complex | **Azure VMs/AKS** | ✅ **None** | ✅✅ **Best** | + +## Troubleshooting + +### SAS Token Issues + +**Error: "Failed to check existence" or "403 Forbidden"** + +This usually means the SAS token lacks required permissions. Verify: +1. ✅ Resource types include: **Service, Container, and Object** (`srt=sco`) + - ❌ Wrong: `srt=c` (container only) + - ✅ Correct: `srt=sco` (service, container, object) +2. ✅ Permissions include at least: **Read, Write, Delete, List, Add, Create** (`sp=rwdlac`) +3. ✅ Token has not expired +4. ✅ `&` characters are escaped as `&` in XML +5. ✅ Container already exists in Azure Blob Storage + +**Error: "Signature did not match"** + +1. Check that `&` characters are properly escaped as `&` in solr.xml +2. Ensure no extra whitespace or line breaks in the token +3. Remove the leading `?` from the token if present +4. Verify the token was copied completely + +### Azure Identity Issues + +**Error: "403 Forbidden" or "AuthorizationFailed"** + +This means your identity lacks the required permissions. Verify: + +1. ✅ **Azure CLI:** You're logged in with `az login` +2. ✅ **RBAC Role:** Identity has "Storage Blob Data Contributor" role +3. ✅ **Scope:** Role is assigned at the correct scope (storage account level) +4. ✅ **Token:** For CLI, run `az account get-access-token --resource https://storage.azure.com/` to verify token + +**Check role assignment:** +```bash +# List all role assignments for the storage account +az role assignment list \ + --scope /subscriptions/YOUR_SUBSCRIPTION_ID/resourceGroups/YOUR_RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts/YOUR_ACCOUNT_NAME \ + --query "[].{Principal:principalName, Role:roleDefinitionName}" -o table +``` + +**Error: "DefaultAzureCredential failed to retrieve token"** + +This means the credential chain couldn't find valid credentials. Check: + +1. **Azure CLI:** Ensure `az login` is successful and not expired +2. **Service Principal:** Verify environment variables or solr.xml credentials are correct +3. **Managed Identity:** Ensure it's enabled on the VM/AKS and has permissions +4. **Token expiry:** Azure CLI tokens expire - re-run `az login` if needed + +**Performance slower than expected:** + +Azure Identity adds ~5-10 seconds overhead for token acquisition. This is normal and expected: +- First operation: ~10-15 seconds (token acquisition) +- Subsequent operations: ~5 seconds (token refresh) +- For large backups (GB/TB), this overhead is negligible + +## Usage + +Once you've configured authentication in `solr.xml`, you can use standard Solr backup/restore commands. + +### Create a Backup + +```bash +# Create a backup of a collection +curl "http://localhost:8983/solr/admin/collections?action=BACKUP&name=my-backup&collection=my-collection&repository=blob&location=/" + +# Example response: +# { +# "responseHeader": {"status": 0, "QTime": 1234}, +# "response": { +# "collection": "my-collection", +# "backupId": 1, +# "indexFileCount": 156, +# "indexSizeMB": 245.5 +# } +# } +``` + +**Parameters:** +- `name` - Backup name (used for restore) +- `collection` - Source collection to backup +- `repository` - Repository name from solr.xml (e.g., `blob`) +- `location` - Path in blob container (use `/` for root, or `/backups/` for subdirectory) + +### Restore from Backup + +```bash +# Restore a backup to a new or existing collection +curl "http://localhost:8983/solr/admin/collections?action=RESTORE&name=my-backup&collection=my-collection-restored&repository=blob&location=/" + +# Example response: +# { +# "responseHeader": {"status": 0, "QTime": 567}, +# "success": {...} +# } +``` + +**Parameters:** +- `name` - Backup name to restore +- `collection` - Target collection name (can be different from original) +- `repository` - Repository name from solr.xml +- `location` - Same path used during backup + +### List Backups + +```bash +# List all backups at a location +curl "http://localhost:8983/solr/admin/collections?action=LISTBACKUP&name=my-backup&repository=blob&location=/" + +# Example response: +# { +# "responseHeader": {"status": 0}, +# "backups": [ +# {"backupId": 1, "indexFileCount": 156, "indexSizeMB": 245.5}, +# {"backupId": 2, "indexFileCount": 158, "indexSizeMB": 247.1} +# ] +# } +``` + +### Delete a Backup + +```bash +# Delete a specific backup +curl "http://localhost:8983/solr/admin/collections?action=DELETEBACKUP&name=my-backup&backupId=1&repository=blob&location=/" +``` + +**Note:** The `location` parameter should be `/` (root of container) or a subdirectory path like `/backups/`. The path must not have a trailing slash except for root. + +### Best Practices + +1. **Naming Convention:** Use descriptive backup names with timestamps + ```bash + curl "...&name=my-collection-2025-10-08&..." + ``` + +2. **Regular Testing:** Periodically test restore operations + ```bash + # Restore to a test collection + curl "...&collection=my-collection-test&..." + ``` + +3. **Multiple Backups:** Keep multiple backup versions + ```bash + # Backups are versioned automatically (backupId) + curl "...action=LISTBACKUP..." # View all versions + ``` + +4. **Monitor Progress:** Use Solr admin UI or check logs + ```bash + tail -f $SOLR_HOME/logs/solr.log | grep -i backup + ``` diff --git a/solr/modules/blob-repository/build.gradle b/solr/modules/blob-repository/build.gradle new file mode 100644 index 000000000000..8e63a84475b3 --- /dev/null +++ b/solr/modules/blob-repository/build.gradle @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: 'java-library' + +description = 'Azure Blob Storage Repository' + +dependencies { + implementation enforcedPlatform("io.netty:netty-bom:4.1.110.Final") + testImplementation enforcedPlatform("io.netty:netty-bom:4.1.110.Final") + implementation platform(project(':platform')) + api(project(':solr:core')) + implementation project(':solr:solrj') + + implementation libs.apache.lucene.core + + // Azure Storage SDK dependencies + implementation libs.azure.storage.blob + implementation libs.azure.identity + implementation libs.azure.core + implementation 'com.azure:azure-storage-common:12.25.0' + + implementation libs.google.guava + implementation libs.slf4j.api + + runtimeOnly libs.fasterxml.woodstox.core + runtimeOnly libs.codehaus.woodstox.stax2api + + testImplementation project(':solr:test-framework') + testImplementation libs.junit.junit + testImplementation libs.commonsio.commonsio + + // Explicit transitive test dependencies for dependency analyzer + testImplementation 'com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.3' + testImplementation 'io.netty:netty-common:4.1.110.Final' + testImplementation 'io.netty:netty-transport:4.1.110.Final' +} \ No newline at end of file diff --git a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobBackupRepository.java b/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobBackupRepository.java new file mode 100644 index 000000000000..54bef83bfefb --- /dev/null +++ b/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobBackupRepository.java @@ -0,0 +1,407 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.blob; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.invoke.MethodHandles; +import java.net.URI; +import java.net.URISyntaxException; +import java.time.Duration; +import java.time.Instant; +import java.util.Collection; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.StrUtils; +import org.apache.solr.core.backup.repository.AbstractBackupRepository; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A concrete implementation of {@link BackupRepository} interface supporting backup/restore of Solr + * indexes to Azure Blob Storage. + */ +public class BlobBackupRepository extends AbstractBackupRepository { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + static final String BLOB_SCHEME = "blob"; + private static final int CHUNK_SIZE = 16 * 1024 * 1024; + + private BlobStorageClient client; + + @Override + public void init(NamedList args) { + super.init(args); + BlobBackupRepositoryConfig backupConfig = new BlobBackupRepositoryConfig(this.config); + + // If a client was already created, close it to avoid any resource leak + if (client != null) { + client.close(); + } + + this.client = backupConfig.buildClient(); + } + + // Method to inject a mock client for testing + public void setClient(BlobStorageClient client) { + this.client = client; + } + + @Override + @SuppressWarnings("unchecked") + public T getConfigProperty(String name) { + return (T) this.config.get(name); + } + + @Override + public URI createURI(String location) { + if (StrUtils.isNullOrEmpty(location)) { + throw new IllegalArgumentException("cannot create URI with an empty location"); + } + + URI result; + try { + if (location.startsWith(BLOB_SCHEME + ":")) { + result = new URI(location); + } else if (location.startsWith("/")) { + result = new URI(BLOB_SCHEME, "", location, null); + } else { + result = new URI(BLOB_SCHEME, "", "/" + location, null); + } + return result; + } catch (URISyntaxException ex) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, ex); + } + } + + @Override + public URI createDirectoryURI(String location) { + if (StrUtils.isNullOrEmpty(location)) { + throw new IllegalArgumentException("cannot create URI with an empty location"); + } + + if (!location.endsWith("/")) { + location += "/"; + } + + return createURI(location); + } + + @Override + public URI resolve(URI baseUri, String... pathComponents) { + if (!BLOB_SCHEME.equalsIgnoreCase(baseUri.getScheme())) { + throw new IllegalArgumentException("URI must begin with 'blob:' scheme"); + } + + String path = baseUri + "/" + String.join("/", pathComponents); + return URI.create(path).normalize(); + } + + @Override + public URI resolveDirectory(URI baseUri, String... pathComponents) { + if (pathComponents.length > 0) { + if (!pathComponents[pathComponents.length - 1].endsWith("/")) { + pathComponents[pathComponents.length - 1] = pathComponents[pathComponents.length - 1] + "/"; + } + } else { + if (!baseUri.toString().endsWith("/")) { + baseUri = URI.create(baseUri + "/"); + } + } + return resolve(baseUri, pathComponents); + } + + @Override + public void createDirectory(URI path) throws IOException { + Objects.requireNonNull(path, "cannot create directory to a null URI"); + + String blobPath = getBlobPath(path); + + if (log.isDebugEnabled()) { + log.debug("Create directory '{}'", blobPath); + } + + try { + client.createDirectory(blobPath); + } catch (BlobException e) { + throw new IOException("Failed to create directory " + blobPath, e); + } + } + + @Override + public void deleteDirectory(URI path) throws IOException { + Objects.requireNonNull(path, "cannot delete directory with a null URI"); + + String blobPath = getBlobPath(path); + + if (log.isDebugEnabled()) { + log.debug("Delete directory '{}'", blobPath); + } + + try { + client.deleteDirectory(blobPath); + } catch (BlobException e) { + throw new IOException("Failed to delete directory " + blobPath, e); + } + } + + @Override + public void delete(URI path, Collection files) throws IOException { + Objects.requireNonNull(path, "cannot delete with a null URI"); + Objects.requireNonNull(files, "cannot delete with a null files collection"); + + String basePath = getBlobPath(path); + // If a file path was passed instead of a directory, use its parent directory as base + try { + if (!client.isDirectory(basePath)) { + int lastSlash = basePath.lastIndexOf('/'); + basePath = lastSlash >= 0 ? basePath.substring(0, lastSlash) : ""; + } + } catch (BlobException e) { + throw new IOException("Failed to check path type for " + basePath, e); + } + + final String baseForPaths = basePath; + Set fullPaths = + files.stream() + .map(file -> (baseForPaths.isEmpty() ? file : baseForPaths + "/" + file)) + .collect(Collectors.toSet()); + + if (log.isDebugEnabled()) { + log.debug("Delete files '{}'", fullPaths); + } + + try { + client.delete(fullPaths); + } catch (BlobException e) { + throw new IOException("Failed to delete files " + fullPaths, e); + } + } + + @Override + public boolean exists(URI path) throws IOException { + Objects.requireNonNull(path, "cannot check existence with a null URI"); + + String blobPath = getBlobPath(path); + + if (log.isDebugEnabled()) { + log.debug("Check existence '{}'", blobPath); + } + + try { + return client.pathExists(blobPath); + } catch (BlobException e) { + throw new IOException("Failed to check existence of " + blobPath, e); + } + } + + @Override + public PathType getPathType(URI path) throws IOException { + Objects.requireNonNull(path, "cannot get path type with a null URI"); + + String blobPath = getBlobPath(path); + + if (log.isDebugEnabled()) { + log.debug("Get path type '{}'", blobPath); + } + + try { + if (client.isDirectory(blobPath)) { + return BackupRepository.PathType.DIRECTORY; + } else { + return BackupRepository.PathType.FILE; + } + } catch (BlobException e) { + throw new IOException("Failed to get path type for " + blobPath, e); + } + } + + @Override + public String[] listAll(URI path) throws IOException { + Objects.requireNonNull(path, "cannot list with a null URI"); + + String blobPath = getBlobPath(path); + + if (log.isDebugEnabled()) { + log.debug("List all '{}'", blobPath); + } + + try { + return client.listDir(blobPath); + } catch (BlobException e) { + throw new IOException("Failed to list directory " + blobPath, e); + } + } + + @Override + public IndexInput openInput(URI dirPath, String fileName, IOContext ctx) throws IOException { + Objects.requireNonNull(dirPath, "cannot open input with a null URI"); + Objects.requireNonNull(fileName, "cannot open input with a null fileName"); + + String base = getBlobPath(dirPath); + String blobPath = base.endsWith("/") ? base + fileName : base + "/" + fileName; + + if (log.isDebugEnabled()) { + log.debug("Open input '{}'", blobPath); + } + + try { + return new BlobIndexInput(blobPath, client, client.length(blobPath)); + } catch (BlobException e) { + throw new IOException("Failed to open input stream for " + blobPath, e); + } + } + + @Override + public OutputStream createOutput(URI path) throws IOException { + Objects.requireNonNull(path, "cannot create output with a null URI"); + + String blobPath = getBlobPath(path); + + if (log.isDebugEnabled()) { + log.debug("Create output '{}'", blobPath); + } + + try { + return client.pushStream(blobPath); + } catch (BlobException e) { + throw new IOException("Failed to create output stream for " + blobPath, e); + } + } + + @Override + public void copyIndexFileFrom( + Directory sourceDir, String sourceFileName, URI dest, String destFileName) + throws IOException { + Objects.requireNonNull(sourceDir, "cannot copy with a null sourceDir"); + Objects.requireNonNull(sourceFileName, "cannot copy with a null sourceFileName"); + Objects.requireNonNull(dest, "cannot copy with a null dest"); + + String destPath = getBlobPath(dest); + + String blobPath = destPath.endsWith("/") ? destPath + destFileName : destPath; + + if (log.isDebugEnabled()) { + log.debug("Copy index file from '{}' to '{}'", sourceFileName, blobPath); + } + + // Ensure destination parent directory exists + String parentDir = + blobPath.contains("/") ? blobPath.substring(0, blobPath.lastIndexOf('/') + 1) : ""; + try { + if (!parentDir.isEmpty()) { + client.createDirectory(parentDir); + } + } catch (BlobException e) { + // ignore failures here; write will surface real issues + } + + try (IndexInput input = sourceDir.openInput(sourceFileName, IOContext.DEFAULT); + OutputStream output = client.pushStream(blobPath)) { + // Copy bytes from IndexInput to OutputStream + byte[] buffer = new byte[8192]; + long remaining = input.length(); + while (remaining > 0) { + int toRead = (int) Math.min(buffer.length, remaining); + input.readBytes(buffer, 0, toRead); + output.write(buffer, 0, toRead); + remaining -= toRead; + } + } catch (BlobException e) { + throw new IOException("Failed to copy file from " + sourceFileName + " to " + blobPath, e); + } + } + + /** + * Copy an index file from specified sourceRepo to the destination directory (i.e. + * restore). + * + * @param sourceDir The source URI hosting the file to be copied. + * @param dest The destination where the file should be copied. + * @throws IOException in case of errors. + */ + @Override + public void copyIndexFileTo( + URI sourceDir, String sourceFileName, Directory dest, String destFileName) + throws IOException { + if (StrUtils.isNullOrEmpty(sourceFileName)) { + throw new IllegalArgumentException("must have a valid source file name to copy"); + } + if (StrUtils.isNullOrEmpty(destFileName)) { + throw new IllegalArgumentException("must have a valid destination file name to copy"); + } + + String basePath = getBlobPath(sourceDir); + String blobPath; + // If sourceDir already points to the file, avoid duplicating the name + if (basePath.endsWith("/" + sourceFileName) + || basePath.equals(sourceFileName) + || basePath.equals("/" + sourceFileName)) { + blobPath = basePath; + } else { + URI filePath = resolve(sourceDir, sourceFileName); + blobPath = getBlobPath(filePath); + } + + Instant start = Instant.now(); + if (log.isDebugEnabled()) { + log.debug("Download started from blob '{}'", blobPath); + } + + try (InputStream inputStream = client.pullStream(blobPath); + IndexOutput indexOutput = dest.createOutput(destFileName, IOContext.DEFAULT)) { + // Copy bytes from InputStream to IndexOutput + byte[] buffer = new byte[CHUNK_SIZE]; + int len; + while ((len = inputStream.read(buffer)) != -1) { + indexOutput.writeBytes(buffer, 0, len); + } + } catch (BlobException e) { + throw new IOException("Failed to copy file from " + blobPath + " to " + destFileName, e); + } + + long timeElapsed = Duration.between(start, Instant.now()).toMillis(); + + if (log.isInfoEnabled()) { + log.info("Download from S3 '{}' finished in {}ms", blobPath, timeElapsed); + } + } + + @Override + public void close() throws IOException { + if (client != null) { + client.close(); + } + } + + private String getBlobPath(URI uri) { + if (!BLOB_SCHEME.equalsIgnoreCase(uri.getScheme())) { + throw new IllegalArgumentException("URI must begin with 'blob:' scheme"); + } + return uri.getPath(); + } +} diff --git a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobBackupRepositoryConfig.java b/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobBackupRepositoryConfig.java new file mode 100644 index 000000000000..59558c3bf7d8 --- /dev/null +++ b/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobBackupRepositoryConfig.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.blob; + +import org.apache.solr.common.util.EnvUtils; +import org.apache.solr.common.util.NamedList; + +/** Class representing the {@code backup} Blob Storage config bundle specified in solr.xml. */ +public class BlobBackupRepositoryConfig { + + public static final String CONTAINER_NAME = "blob.container.name"; + public static final String CONNECTION_STRING = "blob.connection.string"; + public static final String ENDPOINT = "blob.endpoint"; + public static final String ACCOUNT_NAME = "blob.account.name"; + public static final String ACCOUNT_KEY = "blob.account.key"; + public static final String SAS_TOKEN = "blob.sas.token"; + public static final String TENANT_ID = "blob.tenant.id"; + public static final String CLIENT_ID = "blob.client.id"; + public static final String CLIENT_SECRET = "blob.client.secret"; + + private final String containerName; + private final String connectionString; + private final String endpoint; + private final String accountName; + private final String accountKey; + private final String sasToken; + private final String tenantId; + private final String clientId; + private final String clientSecret; + + public BlobBackupRepositoryConfig(NamedList config) { + containerName = getStringConfig(config, CONTAINER_NAME); + connectionString = getStringConfig(config, CONNECTION_STRING); + endpoint = getStringConfig(config, ENDPOINT); + accountName = getStringConfig(config, ACCOUNT_NAME); + accountKey = getStringConfig(config, ACCOUNT_KEY); + sasToken = getStringConfig(config, SAS_TOKEN); + tenantId = getStringConfig(config, TENANT_ID); + clientId = getStringConfig(config, CLIENT_ID); + clientSecret = getStringConfig(config, CLIENT_SECRET); + } + + /** Construct a {@link BlobStorageClient} from the provided config. */ + public BlobStorageClient buildClient() { + return new BlobStorageClient( + containerName, + connectionString, + endpoint, + accountName, + accountKey, + sasToken, + tenantId, + clientId, + clientSecret); + } + + static String getStringConfig(NamedList config, String property) { + String envProp = EnvUtils.getProperty(property); + if (envProp == null) { + Object configProp = config.get(property); + return configProp == null ? null : configProp.toString(); + } else { + return envProp; + } + } +} diff --git a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobException.java b/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobException.java new file mode 100644 index 000000000000..62890aa60efb --- /dev/null +++ b/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobException.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.blob; + +/** + * Generic exception for Blob Storage related failures. Could originate from the {@link + * BlobBackupRepository} or from its underlying {@link BlobStorageClient}. + */ +public class BlobException extends Exception { + public BlobException(String message) { + super(message); + } + + public BlobException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobIndexInput.java b/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobIndexInput.java new file mode 100644 index 000000000000..1938f24f2e3f --- /dev/null +++ b/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobIndexInput.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.blob; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.util.LinkedHashMap; +import java.util.Map; +import org.apache.lucene.store.IndexInput; + +class BlobIndexInput extends IndexInput { + + private static final int DEFAULT_PAGE_SIZE = 512 * 1024; // 512 KB + private static final int MAX_CACHED_PAGES = 128; // ~64 MB at 512 KB pages + + private final String path; + private final BlobStorageClient client; + private final long length; + private final int pageSize; + private final LruPageCache cache; + + private long position = 0L; + private boolean closed = false; + + BlobIndexInput(String path, BlobStorageClient client, long length) { + this(path, client, length, DEFAULT_PAGE_SIZE, MAX_CACHED_PAGES); + } + + BlobIndexInput( + String path, BlobStorageClient client, long length, int pageSize, int maxCachedPages) { + super(path); + this.path = path; + this.client = client; + this.length = length; + this.pageSize = Math.max(4 * 1024, pageSize); + this.cache = new LruPageCache(maxCachedPages); + } + + @Override + public void close() throws IOException { + closed = true; + cache.clear(); + } + + @Override + public long getFilePointer() { + return position; + } + + @Override + public void seek(long pos) throws IOException { + ensureOpen(); + if (pos < 0 || pos > length) { + throw new IOException("Seek position out of bounds: " + pos); + } + + position = pos; + } + + @Override + public long length() { + return length; + } + + @Override + public IndexInput slice(String sliceDescription, long offset, long length) throws IOException { + ensureOpen(); + if (offset < 0 || length < 0 || offset + length > this.length) { + throw new IOException("Slice out of bounds: offset=" + offset + ", length=" + length); + } + + BlobIndexInput slice = + new BlobIndexInput( + getFullSliceDescription(sliceDescription), client, length, pageSize, MAX_CACHED_PAGES); + + slice.position = 0L; + + // Wrap client in a view that remaps range requests by adding base offset + slice.clientViewBaseOffset = this.clientViewBaseOffset + offset; + return slice; + } + + @Override + public byte readByte() throws IOException { + ensureOpen(); + if (position >= length) { + throw new EOFException("End of stream reached"); + } + + byte[] page = getPage(pageIndex(position)); + int inPageOffset = (int) (position % pageSize); + byte value = page[inPageOffset]; + position += 1L; + return value; + } + + @Override + public void readBytes(byte[] b, int offset, int len) throws IOException { + ensureOpen(); + if (len < 0) { + throw new IOException("Length must be non-negative"); + } + + if (position + len > length) { + throw new EOFException("End of stream reached"); + } + + int remaining = len; + while (remaining > 0) { + long pageIdx = pageIndex(position); + byte[] page = getPage(pageIdx); + int inPageOffset = (int) (position % pageSize); + int toCopy = Math.min(remaining, pageSize - inPageOffset); + System.arraycopy(page, inPageOffset, b, offset + (len - remaining), toCopy); + position += toCopy; + remaining -= toCopy; + } + } + + // Internal state for slices: base offset to add to all range requests + private long clientViewBaseOffset = 0L; + + private byte[] getPage(long pageIdx) throws IOException { + byte[] page = cache.get(pageIdx); + if (page != null) { + return page; + } + + long absoluteOffset = clientViewBaseOffset + pageIdx * (long) pageSize; + int bytesToRead = (int) Math.min(pageSize, length - pageIdx * (long) pageSize); + if (bytesToRead <= 0) { + throw new EOFException("End of stream reached"); + } + + page = new byte[bytesToRead]; + try (InputStream in = client.pullRangeStream(path, absoluteOffset, bytesToRead)) { + int readTotal = 0; + while (readTotal < bytesToRead) { + int read = in.read(page, readTotal, bytesToRead - readTotal); + if (read == -1) break; + readTotal += read; + } + + if (readTotal < bytesToRead) { + throw new EOFException( + "End of stream reached: expected " + bytesToRead + " bytes, got " + readTotal); + } + } catch (BlobException e) { + throw new IOException("Failed to fetch range page", e); + } + + cache.put(pageIdx, page); + return page; + } + + private long pageIndex(long pos) { + return pos / pageSize; + } + + private void ensureOpen() throws IOException { + if (closed) { + throw new IOException("IndexInput is closed"); + } + } + + private static final class LruPageCache extends LinkedHashMap { + private final int maxEntries; + + LruPageCache(int maxEntries) { + super(16, 0.75f, true); + this.maxEntries = maxEntries; + } + + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > maxEntries; + } + + @Override + public void clear() { + super.clear(); + } + } +} diff --git a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobNotFoundException.java b/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobNotFoundException.java new file mode 100644 index 000000000000..88e0c41e781d --- /dev/null +++ b/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobNotFoundException.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.blob; + +/** Exception thrown when a blob is not found in Azure Blob Storage. */ +public class BlobNotFoundException extends BlobException { + public BlobNotFoundException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobOutputStream.java b/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobOutputStream.java new file mode 100644 index 000000000000..41a9b3fe0a10 --- /dev/null +++ b/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobOutputStream.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.blob; + +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.specialized.BlockBlobClient; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.invoke.MethodHandles; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.UUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * OutputStream implementation for Azure Blob Storage using block blobs. Supports chunked uploads + * for large files. + */ +public class BlobOutputStream extends OutputStream { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + // 4 MB per block (Azure limit is 100 MB, but 4 MB is more efficient for most use cases) + static final int BLOCK_SIZE = 4 * 1024 * 1024; + + private final BlobClient blobClient; + private final String blobPath; + private volatile boolean closed; + private final ByteBuffer buffer; + private BlockUpload blockUpload; + private boolean committed; + + public BlobOutputStream(BlobClient blobClient, String blobPath) { + this.blobClient = blobClient; + this.blobPath = blobPath; + this.closed = false; + this.buffer = ByteBuffer.allocate(BLOCK_SIZE); + this.blockUpload = null; + this.committed = false; + + if (log.isDebugEnabled()) { + log.debug("Created BlobOutputStream for blobPath '{}'", blobPath); + } + } + + @Override + public void write(int b) throws IOException { + if (closed) { + throw new IOException("Stream closed"); + } + + buffer.put((byte) b); + + // If the buffer is now full, push it to Azure Blob Storage + if (!buffer.hasRemaining()) { + uploadBlock(); + } + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + if (closed) { + throw new IOException("Stream closed"); + } + + if (outOfRange(off, b.length) || len < 0 || outOfRange(off + len, b.length)) { + throw new IndexOutOfBoundsException(); + } else if (len == 0) { + return; + } + + int currentOffset = off; + int lenRemaining = len; + while (buffer.remaining() < lenRemaining) { + int firstPart = buffer.remaining(); + buffer.put(b, currentOffset, firstPart); + uploadBlock(); + + currentOffset += firstPart; + lenRemaining -= firstPart; + } + if (lenRemaining > 0) { + buffer.put(b, currentOffset, lenRemaining); + } + } + + private static boolean outOfRange(int off, int len) { + return off < 0 || off > len; + } + + private void uploadBlock() throws IOException { + int size = buffer.position() - buffer.arrayOffset(); + + if (size == 0) { + // nothing to upload + return; + } + + if (blockUpload == null) { + if (log.isDebugEnabled()) { + log.debug("New block upload for blobPath '{}'", blobPath); + } + blockUpload = newBlockUpload(); + } + + try (ByteArrayInputStream inputStream = + new ByteArrayInputStream(buffer.array(), buffer.arrayOffset(), size)) { + blockUpload.uploadBlock(inputStream, size); + } catch (BlobStorageException e) { + if (blockUpload != null) { + blockUpload.abort(); + if (log.isDebugEnabled()) { + log.debug("Block upload aborted for blobPath '{}'.", blobPath); + } + } + throw new IOException("Failed to upload block", BlobStorageClient.handleBlobException(e)); + } + + // reset the buffer for eventual next write operation + buffer.clear(); + } + + @Override + public void flush() throws IOException { + if (closed) { + throw new IOException("Stream closed"); + } + + // Ensure any buffered data is staged to Azure + if (buffer.position() - buffer.arrayOffset() > 0) { + uploadBlock(); + } + + // Make data visible by committing current block list (idempotent, can be called again on close) + if (blockUpload != null) { + blockUpload.complete(); + blockUpload = null; + committed = true; + } + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + + if (blockUpload != null && blockUpload.aborted) { + blockUpload = null; + closed = true; + return; + } + + if (!committed) { + // Stage any remaining data and commit once + uploadBlock(); + if (blockUpload != null) { + blockUpload.complete(); + blockUpload = null; + committed = true; + } else { + // No data was written; ensure a zero-length blob exists at this path + try { + blobClient.upload(new ByteArrayInputStream(new byte[0]), 0, true); + } catch (BlobStorageException e) { + throw new IOException( + "Failed to create empty blob", BlobStorageClient.handleBlobException(e)); + } + } + } else { + // Already committed via flush. If additional writes occurred after flush, + // there will be a new blockUpload. Commit it to overwrite previous content. + if (blockUpload != null) { + blockUpload.complete(); + blockUpload = null; + } + } + closed = true; + } + + private BlockUpload newBlockUpload() throws IOException { + try { + return new BlockUpload(); + } catch (BlobStorageException e) { + throw new IOException( + "Failed to create block upload", BlobStorageClient.handleBlobException(e)); + } + } + + private class BlockUpload { + private final List blockIds; + private boolean aborted = false; + + public BlockUpload() { + this.blockIds = new ArrayList<>(); + if (log.isDebugEnabled()) { + log.debug("Initiated block upload for blobPath '{}'", blobPath); + } + // Ensure we start with a clean slate; if a blob already exists at this path, + // remove it so that the commit does not fail with BlobAlreadyExists (409). + try { + BlockBlobClient blockBlobClient = blobClient.getBlockBlobClient(); + blockBlobClient.deleteIfExists(); + } catch (BlobStorageException e) { + // Ignore deletion problems here; subsequent stage/commit will surface real issues + } + } + + void uploadBlock(ByteArrayInputStream inputStream, long blockSize) { + if (aborted) { + throw new IllegalStateException( + "Can't upload new blocks on a BlockUpload that was aborted"); + } + + String blockId = + Base64.getEncoder() + .encodeToString(UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8)); + + if (log.isDebugEnabled()) { + log.debug("Uploading block {} for blobPath '{}'", blockId, blobPath); + } + + try { + BlockBlobClient blockBlobClient = blobClient.getBlockBlobClient(); + blockBlobClient.stageBlock(blockId, inputStream, blockSize); + blockIds.add(blockId); + } catch (BlobStorageException e) { + throw new RuntimeException("Failed to upload block", e); + } + } + + /** To be invoked when closing the stream to mark upload is done. */ + void complete() { + if (aborted) { + throw new IllegalStateException("Can't complete a BlockUpload that was aborted"); + } + + if (log.isDebugEnabled()) { + log.debug("Completing block upload for blobPath '{}'", blobPath); + } + + try { + BlockBlobClient blockBlobClient = blobClient.getBlockBlobClient(); + blockBlobClient.commitBlockList(blockIds); + } catch (BlobStorageException e) { + throw new RuntimeException("Failed to commit block list", e); + } + } + + public void abort() { + if (log.isWarnEnabled()) { + log.warn("Aborting block upload for blobPath '{}'", blobPath); + } + + // Azure doesn't have an explicit abort operation for block uploads + // The blocks will remain as uncommitted blocks and will be cleaned up + // by Azure's garbage collection after 7 days + aborted = true; + } + } +} diff --git a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobStorageClient.java b/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobStorageClient.java new file mode 100644 index 000000000000..5ad196caae13 --- /dev/null +++ b/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobStorageClient.java @@ -0,0 +1,549 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.blob; + +import com.azure.core.credential.TokenCredential; +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.blob.models.BlobItem; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.ListBlobsOptions; +import com.google.common.annotations.VisibleForTesting; +import java.io.ByteArrayInputStream; +import java.io.FilterInputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.invoke.MethodHandles; +import java.util.Collection; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.solr.common.util.ResumableInputStream; +import org.apache.solr.common.util.StrUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Creates a {@link BlobServiceClient} for communicating with Azure Blob Storage. Utilizes the + * default Azure credential provider chain. + */ +public class BlobStorageClient { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + static final String BLOB_FILE_PATH_DELIMITER = "/"; + + private final BlobContainerClient containerClient; + + BlobStorageClient( + String containerName, + String connectionString, + String endpoint, + String accountName, + String accountKey, + String sasToken, + String tenantId, + String clientId, + String clientSecret) { + this( + createInternalClient( + connectionString, + endpoint, + accountName, + accountKey, + sasToken, + tenantId, + clientId, + clientSecret), + containerName); + } + + @VisibleForTesting + BlobStorageClient(BlobServiceClient blobServiceClient, String containerName) { + this.containerClient = blobServiceClient.getBlobContainerClient(containerName); + try { + containerClient.create(); + } catch (BlobStorageException e) { + if (e.getStatusCode() != 409) { + throw e; + } + } + } + + private static BlobServiceClient createInternalClient( + String connectionString, + String endpoint, + String accountName, + String accountKey, + String sasToken, + String tenantId, + String clientId, + String clientSecret) { + + BlobServiceClientBuilder builder = new BlobServiceClientBuilder(); + // Use default HTTP client (Netty) as provided by azure-core-http-netty + + if (StrUtils.isNotNullOrEmpty(connectionString)) { + builder.connectionString(connectionString); + } else if (StrUtils.isNotNullOrEmpty(endpoint)) { + builder.endpoint(endpoint); + if (StrUtils.isNotNullOrEmpty(accountName) && StrUtils.isNotNullOrEmpty(accountKey)) { + builder.credential( + new com.azure.storage.common.StorageSharedKeyCredential(accountName, accountKey)); + } else if (StrUtils.isNotNullOrEmpty(sasToken)) { + builder.sasToken(sasToken); + } else { + // Use default Azure credential provider chain + TokenCredential credential = new DefaultAzureCredentialBuilder().tenantId(tenantId).build(); + builder.credential(credential); + } + } else { + throw new IllegalArgumentException("Either connectionString or endpoint must be provided"); + } + + return builder.buildClient(); + } + + /** Create a directory in Blob Storage, if it does not already exist. */ + void createDirectory(String path) throws BlobException { + String sanitizedDirPath = sanitizedDirPath(path); + + // Only create the directory if it does not already exist + if (!pathExists(sanitizedDirPath)) { + String parent = getParentDirectory(sanitizedDirPath); + // Stop at root + if (!parent.isEmpty() && !parent.equals(BLOB_FILE_PATH_DELIMITER)) { + createDirectory(parent); + } + + try { + // Create empty blob and mark it as a directory via metadata + BlobClient blobClient = containerClient.getBlobClient(sanitizedDirPath); + blobClient.upload(new ByteArrayInputStream(new byte[0]), 0, true); + java.util.Map metadata = new java.util.HashMap<>(); + metadata.put("hdi_isfolder", "true"); + blobClient.setMetadata(metadata); + } catch (BlobStorageException e) { + throw handleBlobException(e); + } + } + } + + /** Delete files from Blob Storage. Missing files are ignored (idempotent delete). */ + void delete(Collection paths) throws BlobException { + Set entries = new HashSet<>(); + for (String path : paths) { + entries.add(sanitizedFilePath(path)); + } + deleteBlobs(entries); + } + + /** Delete directory, all the files and subdirectories from Blob Storage. */ + void deleteDirectory(String path) throws BlobException { + path = sanitizedDirPath(path); + + // Get all the files and subdirectories + Set entries = listAll(path); + if (pathExists(path)) { + entries.add(path); + } + + deleteBlobs(entries); + } + + /** List all the files and subdirectories directly under given path. */ + String[] listDir(String path) throws BlobException { + path = sanitizedDirPath(path); + + try { + ListBlobsOptions options = new ListBlobsOptions().setPrefix(path).setMaxResultsPerPage(1000); + + final String finalPath = path; // Make path effectively final for lambda + return containerClient.listBlobs(options, null).stream() + .map(BlobItem::getName) + .filter(s -> s.startsWith(finalPath)) + .map(s -> s.substring(finalPath.length())) + .filter(s -> !s.isEmpty()) + .filter( + s -> { + int slashIndex = s.indexOf(BLOB_FILE_PATH_DELIMITER); + return slashIndex == -1 || slashIndex == s.length() - 1; + }) + .toArray(String[]::new); + } catch (BlobStorageException e) { + throw handleBlobException(e); + } + } + + /** Check if path exists. */ + boolean pathExists(String path) throws BlobException { + final String blobPath = sanitizedPath(path); + + // for root return true + if (blobPath.isEmpty() || BLOB_FILE_PATH_DELIMITER.equals(blobPath)) { + return true; + } + + try { + BlobClient blobClient = containerClient.getBlobClient(blobPath); + return blobClient.exists(); + } catch (BlobStorageException e) { + throw handleBlobException(e); + } + } + + /** Check if path is directory. */ + boolean isDirectory(String path) throws BlobException { + final String dirPrefix = sanitizedDirPath(path); + + try { + // First, if there are any child blobs under this prefix, it's a directory + ListBlobsOptions options = + new ListBlobsOptions().setPrefix(dirPrefix).setMaxResultsPerPage(1); + if (containerClient.listBlobs(options, null).iterator().hasNext()) { + return true; + } + + // Otherwise, check if an empty blob exactly named with the trailing slash exists + BlobClient markerClient = containerClient.getBlobClient(dirPrefix); + if (markerClient.exists()) { + long size = markerClient.getProperties().getBlobSize(); + if (size == 0) { + // zero-byte marker with name ending in '/' is a directory + return true; + } + // If it's a non-zero blob at a name with '/', treat conservatively as file + java.util.Map md = markerClient.getProperties().getMetadata(); + return md != null && md.containsKey("hdi_isfolder"); + } + + return false; + } catch (BlobStorageException e) { + throw handleBlobException(e); + } + } + + /** Get length of file in bytes. */ + long length(String path) throws BlobException { + String blobPath = sanitizedFilePath(path); + try { + BlobClient blobClient = containerClient.getBlobClient(blobPath); + return blobClient.getProperties().getBlobSize(); + } catch (BlobStorageException e) { + throw handleBlobException(e); + } + } + + /** Open a new {@link InputStream} to file for read. */ + InputStream pullStream(String path) throws BlobException { + final String blobPath = sanitizedFilePath(path); + + try { + BlobClient blobClient = containerClient.getBlobClient(blobPath); + final long contentLength = blobClient.getProperties().getBlobSize(); + + InputStream initial = new IdempotentCloseInputStream(blobClient.openInputStream()); + + return new ResumableInputStream( + initial, + bytesRead -> { + if (contentLength > 0 && bytesRead >= contentLength) { + return null; + } + try { + long remaining = + contentLength > 0 ? Math.max(0, contentLength - bytesRead) : Long.MAX_VALUE; + return pullRangeStream(path, bytesRead, remaining); + } catch (BlobException e) { + // ResumableInputStream supplier cannot throw checked exceptions + throw new RuntimeException(e); + } + }); + } catch (BlobStorageException e) { + throw handleBlobException(e); + } + } + + /** Open a ranged {@link InputStream} to file for read from offset for length bytes. */ + InputStream pullRangeStream(String path, long offset, long length) throws BlobException { + final String blobPath = sanitizedFilePath(path); + try { + BlobClient blobClient = containerClient.getBlobClient(blobPath); + com.azure.storage.blob.models.BlobRange range = + new com.azure.storage.blob.models.BlobRange(offset, length); + return new IdempotentCloseInputStream(blobClient.openInputStream(range, null)); + } catch (BlobStorageException e) { + throw handleBlobException(e); + } + } + + /** Wrapper that makes close() idempotent (second close is a no-op). */ + private static final class IdempotentCloseInputStream extends FilterInputStream { + private boolean closed; + + IdempotentCloseInputStream(InputStream in) { + super(in); + this.closed = false; + } + + @Override + public int read() throws java.io.IOException { + if (closed) { + throw new java.io.IOException("Stream is already closed"); + } + try { + return super.read(); + } catch (RuntimeException re) { + if (isAlreadyClosed(re)) { + throw new java.io.IOException("Stream is already closed", re); + } + throw re; + } + } + + @Override + public int read(byte[] b, int off, int len) throws java.io.IOException { + if (closed) { + throw new java.io.IOException("Stream is already closed"); + } + try { + return super.read(b, off, len); + } catch (RuntimeException re) { + if (isAlreadyClosed(re)) { + throw new java.io.IOException("Stream is already closed", re); + } + throw re; + } + } + + @Override + public void close() throws java.io.IOException { + if (closed) { + return; + } + try { + super.close(); + } catch (java.io.IOException e) { + String msg = e.getMessage(); + if (msg == null || !msg.toLowerCase(java.util.Locale.ROOT).contains("already closed")) { + throw e; + } + // swallow "already closed" to make close idempotent + } finally { + closed = true; + } + } + + @Override + public long skip(long n) throws java.io.IOException { + if (closed) { + throw new java.io.IOException("Stream is already closed"); + } + if (n <= 0) { + return 0L; + } + long remaining = n; + byte[] discard = new byte[8192]; + try { + while (remaining > 0) { + int toRead = (int) Math.min(discard.length, remaining); + int read = super.read(discard, 0, toRead); + if (read < 0) { + break; + } + remaining -= read; + } + return n - remaining; + } catch (RuntimeException re) { + // Normalize runtime issues from Azure's stream into IOExceptions so upper layers can resume + throw new java.io.IOException(re); + } + } + + private static boolean isAlreadyClosed(Throwable t) { + String msg = t.getMessage(); + return msg != null && msg.toLowerCase(java.util.Locale.ROOT).contains("already closed"); + } + } + + /** Open a new {@link OutputStream} to file for write. */ + OutputStream pushStream(String path) throws BlobException { + path = sanitizedFilePath(path); + + if (!parentDirectoryExist(path)) { + // Auto-create missing parent directory to mirror Azure's virtual directory semantics + String parentDirectory = getParentDirectory(path); + if (!parentDirectory.isEmpty() && !parentDirectory.equals(BLOB_FILE_PATH_DELIMITER)) { + createDirectory(parentDirectory); + } + } + + try { + BlobClient blobClient = containerClient.getBlobClient(path); + return new BlobOutputStream(blobClient, path); + } catch (BlobStorageException e) { + throw handleBlobException(e); + } + } + + /** Close the client. */ + void close() { + // Azure SDK clients don't need explicit closing + } + + @VisibleForTesting + void deleteContainerForTests() { + try { + containerClient.delete(); + } catch (BlobStorageException e) { + // Ignore not found + if (e.getStatusCode() != 404) { + throw e; + } + } + } + + private Collection deleteBlobs(Collection paths) throws BlobException { + try { + return deleteBlobs(paths, 1000); // Azure supports batch delete + } catch (BlobStorageException e) { + throw handleBlobException(e); + } + } + + @VisibleForTesting + Collection deleteBlobs(Collection entries, int batchSize) throws BlobException { + Set deletedPaths = new HashSet<>(); + + for (String path : entries) { + try { + BlobClient blobClient = containerClient.getBlobClient(path); + boolean existed = blobClient.deleteIfExists(); + if (existed) { + deletedPaths.add(path); + } + } catch (BlobStorageException e) { + if (e.getStatusCode() == 404) { + // ignore missing + continue; + } + throw new BlobException("Could not delete blob with path: " + path, e); + } + } + + return deletedPaths; + } + + private Set listAll(String path) throws BlobException { + String prefix = sanitizedDirPath(path); + + try { + ListBlobsOptions options = + new ListBlobsOptions().setPrefix(prefix).setMaxResultsPerPage(1000); + + return containerClient.listBlobs(options, null).stream() + .map(BlobItem::getName) + .filter(s -> s.startsWith(prefix)) + .collect(Collectors.toSet()); + } catch (BlobStorageException e) { + throw handleBlobException(e); + } + } + + private boolean parentDirectoryExist(String path) throws BlobException { + String parentDirectory = getParentDirectory(path); + + if (parentDirectory.isEmpty() || parentDirectory.equals(BLOB_FILE_PATH_DELIMITER)) { + return true; + } + + return pathExists(parentDirectory); + } + + private String getParentDirectory(String path) { + if (!path.contains(BLOB_FILE_PATH_DELIMITER)) { + return ""; + } + + int fromEnd = path.length() - 1; + if (path.endsWith(BLOB_FILE_PATH_DELIMITER)) { + fromEnd -= 1; + } + return fromEnd > 0 + ? path.substring(0, path.lastIndexOf(BLOB_FILE_PATH_DELIMITER, fromEnd) + 1) + : ""; + } + + /** Ensures path adheres to some rules: -Doesn't start with a leading slash */ + String sanitizedPath(String path) throws BlobException { + String sanitizedPath = path.trim(); + // Remove all leading slashes so that blob names never start with '/' + while (sanitizedPath.startsWith(BLOB_FILE_PATH_DELIMITER)) { + sanitizedPath = sanitizedPath.substring(1).trim(); + } + return sanitizedPath; + } + + /** Ensures file path adheres to some rules */ + String sanitizedFilePath(String path) throws BlobException { + String sanitizedPath = sanitizedPath(path); + + if (sanitizedPath.endsWith(BLOB_FILE_PATH_DELIMITER)) { + throw new BlobException("Invalid Path. Path for file can't end with '/'"); + } + + if (sanitizedPath.isEmpty()) { + throw new BlobException("Invalid Path. Path cannot be empty"); + } + + return sanitizedPath; + } + + /** Ensures directory path adheres to some rules */ + String sanitizedDirPath(String path) throws BlobException { + String sanitizedPath = sanitizedPath(path); + + if (!sanitizedPath.endsWith(BLOB_FILE_PATH_DELIMITER)) { + sanitizedPath += BLOB_FILE_PATH_DELIMITER; + } + + return sanitizedPath; + } + + /** Handle Azure Blob Storage exceptions */ + static BlobException handleBlobException(BlobStorageException e) { + String errMessage = + String.format( + Locale.ROOT, + "Azure Blob Storage error: [statusCode=%s] [errorCode=%s] [message=%s]", + e.getStatusCode(), + e.getErrorCode(), + e.getMessage()); + + log.error(errMessage); + + if (e.getStatusCode() == 404) { + return new BlobNotFoundException(errMessage, e); + } else { + return new BlobException(errMessage, e); + } + } +} diff --git a/solr/modules/blob-repository/src/java/org/apache/solr/blob/package-info.java b/solr/modules/blob-repository/src/java/org/apache/solr/blob/package-info.java new file mode 100644 index 000000000000..bb93394a314a --- /dev/null +++ b/solr/modules/blob-repository/src/java/org/apache/solr/blob/package-info.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Azure Blob Storage backup repository implementation for Apache Solr. + * + *

This package provides a {@link org.apache.solr.blob.BlobBackupRepository} implementation that + * enables Solr to store and retrieve backup data from Azure Blob Storage. + * + *

The repository supports various Azure authentication methods including: + * + *

    + *
  • Connection strings + *
  • Account name and key + *
  • SAS tokens + *
  • Azure Identity (Managed Identity, Service Principal) + *
+ * + *

Key components: + * + *

    + *
  • {@link org.apache.solr.blob.BlobBackupRepository} - Main repository implementation + *
  • {@link org.apache.solr.blob.BlobStorageClient} - Azure Blob Storage client wrapper + *
  • {@link org.apache.solr.blob.BlobBackupRepositoryConfig} - Configuration management + *
+ * + * @see Azure Blob Storage + * Documentation + */ +package org.apache.solr.blob; diff --git a/solr/modules/blob-repository/src/test-files/conf/schema.xml b/solr/modules/blob-repository/src/test-files/conf/schema.xml new file mode 100644 index 000000000000..a3a7cc465c27 --- /dev/null +++ b/solr/modules/blob-repository/src/test-files/conf/schema.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + id + diff --git a/solr/modules/blob-repository/src/test-files/conf/solrconfig.xml b/solr/modules/blob-repository/src/test-files/conf/solrconfig.xml new file mode 100644 index 000000000000..853ba6562416 --- /dev/null +++ b/solr/modules/blob-repository/src/test-files/conf/solrconfig.xml @@ -0,0 +1,51 @@ + + + + + + + + + ${solr.data.dir:} + + + + + ${tests.luceneMatchVersion:LATEST} + + + + ${solr.commitwithin.softcommit:true} + + + + + + + explicit + true + text + + + + + +: + + diff --git a/solr/modules/blob-repository/src/test-files/log4j2.xml b/solr/modules/blob-repository/src/test-files/log4j2.xml new file mode 100644 index 000000000000..528299e3e0bd --- /dev/null +++ b/solr/modules/blob-repository/src/test-files/log4j2.xml @@ -0,0 +1,40 @@ + + + + + + + + + %maxLen{%-4r %-5p (%t) [%notEmpty{n:%X{node_name}}%notEmpty{ c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{ + =>%ex{short}}}{10240}%n + + + + + + + + + + + + + + + diff --git a/solr/modules/blob-repository/src/test/org/apache/solr/blob/AbstractBlobClientTest.java b/solr/modules/blob-repository/src/test/org/apache/solr/blob/AbstractBlobClientTest.java new file mode 100644 index 000000000000..b28833e4bf68 --- /dev/null +++ b/solr/modules/blob-repository/src/test/org/apache/solr/blob/AbstractBlobClientTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.blob; + +import com.azure.core.http.HttpClient; +import com.azure.core.http.netty.NettyAsyncHttpClientBuilder; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.BlobServiceClientBuilder; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; +import org.apache.solr.SolrTestCaseJ4; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; +import reactor.netty.resources.ConnectionProvider; + +/** Abstract class for tests with Azure Blob Storage emulator. */ +public class AbstractBlobClientTest extends SolrTestCaseJ4 { + + protected String containerName; + + @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); + + BlobStorageClient client; + private static String connectionString; + private EventLoopGroup eventLoopGroup; + private ConnectionProvider connectionProvider; + protected org.apache.solr.client.solrj.cloud.SocketProxy proxy; + + @Before + public void setUpClient() throws Exception { + setAzureTestCredentials(); + + // Disable Netty Flight Recorder to avoid Security Manager issues + // Keep default Netty client; OkHttp dependency not present + + // Use Azurite connection string for local testing + connectionString = + "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;"; + + // Build a Netty HTTP client with isolated resources we can shut down after tests + connectionProvider = ConnectionProvider.create("solr-azure-test"); + eventLoopGroup = new NioEventLoopGroup(1); + + // Put a proxy in front of Azurite to simulate connection loss like S3 tests + proxy = new org.apache.solr.client.solrj.cloud.SocketProxy(); + proxy.open(new java.net.URI(getBlobServiceUrl())); + + HttpClient httpClient = + new NettyAsyncHttpClientBuilder() + .connectionProvider(connectionProvider) + .eventLoopGroup(eventLoopGroup) + .build(); + + // Route Blob endpoint through the proxy by adjusting the connection string + String proxiedConn = connectionString.replace(":10000", ":" + proxy.getListenPort()); + BlobServiceClient blobServiceClient = + new BlobServiceClientBuilder() + .connectionString(proxiedConn) + .httpClient(httpClient) + .buildClient(); + + containerName = "test-" + java.util.UUID.randomUUID(); + client = new BlobStorageClient(blobServiceClient, containerName); + } + + /** + * Set up Azure test credentials to avoid using real Azure credentials during testing. Similar to + * how S3 tests use ProfileFileSystemSetting to avoid polluting the test environment. + */ + public static void setAzureTestCredentials() { + // Set test Azure credentials to avoid using real credentials + System.setProperty("AZURE_CLIENT_ID", "test-client-id"); + System.setProperty("AZURE_TENANT_ID", "test-tenant-id"); + System.setProperty("AZURE_CLIENT_SECRET", "test-client-secret"); + + // Set Azurite-specific environment variables + System.setProperty( + "AZURE_STORAGE_CONNECTION_STRING", + "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;"); + } + + @After + public void tearDownClient() { + if (client != null) { + try { + client.deleteContainerForTests(); + } catch (Throwable ignored) { + } + client.close(); + } + if (proxy != null) { + proxy.close(); + proxy = null; + } + try { + reactor.core.scheduler.Schedulers.shutdownNow(); + reactor.core.scheduler.Schedulers.resetFactory(); + } catch (Throwable ignored) { + } + + // Dispose custom Netty resources to prevent leaked threads + try { + if (connectionProvider != null) { + connectionProvider.disposeLater().block(); + } + } catch (Throwable ignored) { + } + try { + if (eventLoopGroup != null) { + eventLoopGroup.shutdownGracefully(0, 2, TimeUnit.SECONDS).awaitUninterruptibly(3000); + } + } catch (Throwable ignored) { + } + } + + /** Simulate a connection loss on the proxy similar to S3 tests. */ + void initiateBlobConnectionLoss() throws BlobException { + if (proxy != null) { + proxy.halfClose(); + } + } + + @org.junit.AfterClass + public static void afterAll() { + try { + reactor.core.scheduler.Schedulers.shutdownNow(); + reactor.core.scheduler.Schedulers.resetFactory(); + } catch (Throwable ignored) { + } + } + + /** + * Helper method to push a string to Azure Blob Storage. + * + * @param path Destination path in blob storage. + * @param content Arbitrary content for the test. + */ + void pushContent(String path, String content) throws BlobException { + pushContent(path, content.getBytes(StandardCharsets.UTF_8)); + } + + void pushContent(String path, byte[] content) throws BlobException { + try (OutputStream output = client.pushStream(path)) { + output.write(content); + } catch (IOException e) { + throw new BlobException("Failed to write content", e); + } + } + + /** Get the connection string for tests that need direct access to the blob service. */ + static String getConnectionString() { + return connectionString; + } + + /** Get the blob service URL for tests that need direct access. */ + String getBlobServiceUrl() { + return "http://localhost:10000"; + } +} diff --git a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobBackupRepositoryTest.java b/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobBackupRepositoryTest.java new file mode 100644 index 000000000000..690808a83557 --- /dev/null +++ b/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobBackupRepositoryTest.java @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.blob; + +import static org.apache.solr.blob.BlobBackupRepository.BLOB_SCHEME; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.commons.io.file.PathUtils; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.junit.Before; +import org.junit.Test; + +public class BlobBackupRepositoryTest extends AbstractBlobClientTest { + + private BlobBackupRepository repository; + + protected static final String CONTAINER_NAME = "test-container"; + + protected Class getRepositoryClass() { + return BlobBackupRepository.class; + } + + protected BackupRepository getRepository() { + return repository; + } + + protected URI getBaseUri() { + return URI.create(BLOB_SCHEME + ":/"); + } + + @Before + public void setUp() throws Exception { + super.setUp(); + + NamedList config = new NamedList<>(); + config.add("blob.container.name", CONTAINER_NAME); + config.add("blob.connection.string", getConnectionString()); + + // Use a repository that avoids creating its own Azure client (which leaks Netty threads) + // and instead inject the pre-configured client from AbstractBlobClientTest. + repository = + new BlobBackupRepository() { + @Override + public void init(NamedList args) { + // Only capture config; avoid building a new client inside init + this.config = args; + // Inject the already-initialized client that uses isolated Netty resources + setClient(BlobBackupRepositoryTest.this.client); + } + }; + repository.init(config); + } + + @Test + public void testCreateDirectory() throws IOException { + URI dirUri = getBaseUri().resolve("test-dir/"); + repository.createDirectory(dirUri); + assertTrue("Directory should exist", repository.exists(dirUri)); + assertEquals( + "Should be a directory", + BackupRepository.PathType.DIRECTORY, + repository.getPathType(dirUri)); + } + + @Test + public void testCreateFile() throws IOException { + URI fileUri = getBaseUri().resolve("test-file.txt"); + String content = "Hello, Azure Blob Storage!"; + + try (OutputStream output = repository.createOutput(fileUri)) { + output.write(content.getBytes(StandardCharsets.UTF_8)); + } + + assertTrue("File should exist", repository.exists(fileUri)); + assertEquals( + "Should be a file", BackupRepository.PathType.FILE, repository.getPathType(fileUri)); + } + + @Test + public void testReadWriteFile() throws IOException { + URI fileUri = getBaseUri().resolve("read-write-test.txt"); + String originalContent = "Test content for read/write operations"; + + // Write content + try (OutputStream output = repository.createOutput(fileUri)) { + output.write(originalContent.getBytes(StandardCharsets.UTF_8)); + } + + // Read content + try (IndexInput input = + repository.openInput(getBaseUri(), "read-write-test.txt", IOContext.DEFAULT)) { + byte[] buffer = new byte[1024]; + input.readBytes(buffer, 0, (int) input.length()); + String readContent = new String(buffer, 0, (int) input.length(), StandardCharsets.UTF_8); + assertEquals("Content should match", originalContent, readContent); + } + } + + @Test + public void testDeleteFile() throws IOException { + URI fileUri = getBaseUri().resolve("delete-test.txt"); + String content = "File to be deleted"; + + // Create file + try (OutputStream output = repository.createOutput(fileUri)) { + output.write(content.getBytes(StandardCharsets.UTF_8)); + } + + assertTrue("File should exist before deletion", repository.exists(fileUri)); + + // Delete file + repository.delete(fileUri, java.util.Arrays.asList("delete-test.txt")); + + assertFalse("File should not exist after deletion", repository.exists(fileUri)); + } + + @Test + public void testDeleteDirectory() throws IOException { + URI dirUri = getBaseUri().resolve("delete-dir/"); + URI fileUri = dirUri.resolve("nested-file.txt"); + + // Create directory and file + repository.createDirectory(dirUri); + try (OutputStream output = repository.createOutput(fileUri)) { + output.write("Nested file content".getBytes(StandardCharsets.UTF_8)); + } + + assertTrue("Directory should exist", repository.exists(dirUri)); + assertTrue("File should exist", repository.exists(fileUri)); + + // Delete directory + repository.deleteDirectory(dirUri); + + assertFalse("Directory should not exist after deletion", repository.exists(dirUri)); + assertFalse("File should not exist after deletion", repository.exists(fileUri)); + } + + @Test + public void testListDirectory() throws IOException { + URI dirUri = getBaseUri().resolve("list-test/"); + repository.createDirectory(dirUri); + + // Create some files + String[] fileNames = {"file1.txt", "file2.txt", "subdir/"}; + for (String fileName : fileNames) { + URI fileUri = dirUri.resolve(fileName); + if (fileName.endsWith("/")) { + repository.createDirectory(fileUri); + } else { + try (OutputStream output = repository.createOutput(fileUri)) { + output.write(("Content of " + fileName).getBytes(StandardCharsets.UTF_8)); + } + } + } + + String[] listedFiles = repository.listAll(dirUri); + assertEquals("Should list all files and directories", fileNames.length, listedFiles.length); + + for (String fileName : fileNames) { + boolean found = false; + for (String listedFile : listedFiles) { + if (fileName.equals(listedFile)) { + found = true; + break; + } + } + assertTrue("Should find file: " + fileName, found); + } + } + + @Test + public void testCopyFileFromDirectory() throws IOException { + // Create a temporary directory with a file + Path tempDir = Files.createTempDirectory("blob-test"); + Path tempFile = tempDir.resolve("source-file.txt"); + String content = "Source file content"; + Files.write(tempFile, content.getBytes(StandardCharsets.UTF_8)); + + try { + Directory sourceDir = new org.apache.lucene.store.MMapDirectory(tempDir); + URI destUri = getBaseUri().resolve("copied-file.txt"); + + repository.copyFileFrom(sourceDir, "source-file.txt", destUri); + + assertTrue("Copied file should exist", repository.exists(destUri)); + + // Verify content + try (IndexInput input = + repository.openInput(getBaseUri(), "copied-file.txt", IOContext.DEFAULT)) { + byte[] buffer = new byte[1024]; + input.readBytes(buffer, 0, (int) input.length()); + String readContent = new String(buffer, 0, (int) input.length(), StandardCharsets.UTF_8); + assertEquals("Content should match", content, readContent); + } + + sourceDir.close(); + } finally { + PathUtils.deleteDirectory(tempDir); + } + } + + @Test + public void testCopyFileToDirectory() throws IOException { + // Create a file in blob storage + URI sourceUri = getBaseUri().resolve("source-file.txt"); + String content = "Source file content"; + + try (OutputStream output = repository.createOutput(sourceUri)) { + output.write(content.getBytes(StandardCharsets.UTF_8)); + } + + // Create a temporary directory + Path tempDir = Files.createTempDirectory("blob-test"); + + try { + Directory destDir = new org.apache.lucene.store.MMapDirectory(tempDir); + + repository.copyFileTo(sourceUri, "source-file.txt", destDir); + + Path destFile = tempDir.resolve("source-file.txt"); + assertTrue("Destination file should exist", Files.exists(destFile)); + + String readContent = Files.readString(destFile, StandardCharsets.UTF_8); + assertEquals("Content should match", content, readContent); + + destDir.close(); + } finally { + PathUtils.deleteDirectory(tempDir); + } + } + + @Test + public void testIndexInputOutput() throws IOException { + URI fileUri = getBaseUri().resolve("index-test.txt"); + String content = "Test content for index input/output"; + + // Write using IndexOutput + try (OutputStream output = repository.createOutput(fileUri)) { + output.write(content.getBytes(StandardCharsets.UTF_8)); + } + + // Read using IndexInput + try (IndexInput input = + repository.openInput(getBaseUri(), "index-test.txt", IOContext.DEFAULT)) { + byte[] buffer = new byte[(int) input.length()]; + input.readBytes(buffer, 0, buffer.length); + String readContent = new String(buffer, StandardCharsets.UTF_8); + assertEquals("Content should match", content, readContent); + } + } + + @Test + public void testChecksumVerification() throws IOException { + // Create a file with checksum + URI fileUri = getBaseUri().resolve("checksum-test.txt"); + String content = "Test content for checksum verification"; + + try (OutputStream output = repository.createOutput(fileUri)) { + output.write(content.getBytes(StandardCharsets.UTF_8)); + // Write a simple footer for testing + output.write("FOOTER".getBytes(StandardCharsets.UTF_8)); + } + + // Verify content (skip checksum verification for this simple test) + try (IndexInput input = + repository.openInput(getBaseUri(), "checksum-test.txt", IOContext.DEFAULT)) { + byte[] buffer = new byte[1024]; + input.readBytes(buffer, 0, (int) input.length()); + String readContent = new String(buffer, 0, (int) input.length(), StandardCharsets.UTF_8); + assertTrue("Content should contain original text", readContent.contains(content)); + } + } + + /** + * Provide a base {@link BackupRepository} configuration for use by any tests that call {@link + * BackupRepository#init(NamedList)} explicitly. + * + *

Useful for setting configuration properties required for specific BackupRepository + * implementations. + */ + protected NamedList getBaseBackupRepositoryConfiguration() { + NamedList config = new NamedList<>(); + config.add("blob.container.name", CONTAINER_NAME); + config.add("blob.connection.string", getConnectionString()); + return config; + } + + @Test + public void testCanReadProvidedConfigValues() throws Exception { + final NamedList config = getBaseBackupRepositoryConfiguration(); + config.add("configKey1", "configVal1"); + config.add("configKey2", "configVal2"); + config.add("location", "foo"); + try (BackupRepository repo = getRepository()) { + repo.init(config); + assertEquals("configVal1", repo.getConfigProperty("configKey1")); + assertEquals("configVal2", repo.getConfigProperty("configKey2")); + } + } + + @Test + public void testCanChooseDefaultOrOverrideLocationValue() throws Exception { + final NamedList config = getBaseBackupRepositoryConfiguration(); + config.add("location", "foo"); + try (BackupRepository repo = getRepository()) { + repo.init(config); + assertEquals("foo", repo.getConfigProperty("location")); + } + } + + @Override + public void tearDown() throws Exception { + if (repository != null) { + repository.close(); + } + super.tearDown(); + } +} diff --git a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobIncrementalBackupTest.java b/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobIncrementalBackupTest.java new file mode 100644 index 000000000000..d3bce0b1ab88 --- /dev/null +++ b/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobIncrementalBackupTest.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.blob; + +import java.nio.charset.StandardCharsets; +import org.junit.Test; + +public class BlobIncrementalBackupTest extends AbstractBlobClientTest { + + @Test + public void testIncrementalBackup() throws Exception { + String backupPath = "incremental-backup-test/"; + + // Create initial backup + createBackup(backupPath + "backup1/", "Initial backup content"); + + // Create incremental backup + createBackup(backupPath + "backup2/", "Incremental backup content"); + + // Verify both backups exist + assertTrue("Initial backup should exist", client.pathExists(backupPath + "backup1/")); + assertTrue("Incremental backup should exist", client.pathExists(backupPath + "backup2/")); + } + + @Test + public void testBackupWithMultipleFiles() throws Exception { + String backupPath = "multi-file-backup-test/"; + + // Create backup with multiple files + String[] files = {"file1.txt", "file2.txt", "file3.txt"}; + String[] contents = {"Content 1", "Content 2", "Content 3"}; + + for (int i = 0; i < files.length; i++) { + pushContent(backupPath + files[i], contents[i]); + } + + // Verify all files exist + for (String file : files) { + assertTrue("File should exist: " + file, client.pathExists(backupPath + file)); + } + } + + @Test + public void testBackupWithNestedDirectories() throws Exception { + String backupPath = "nested-backup-test/"; + + // Create nested directory structure + String[] dirs = { + backupPath + "level1/", backupPath + "level1/level2/", backupPath + "level1/level2/level3/" + }; + + for (String dir : dirs) { + client.createDirectory(dir); + } + + // Add files at different levels + pushContent(backupPath + "root-file.txt", "Root file content"); + pushContent(backupPath + "level1/mid-file.txt", "Mid file content"); + pushContent(backupPath + "level1/level2/level3/deep-file.txt", "Deep file content"); + + // Verify structure + assertTrue("Root file should exist", client.pathExists(backupPath + "root-file.txt")); + assertTrue("Mid file should exist", client.pathExists(backupPath + "level1/mid-file.txt")); + assertTrue( + "Deep file should exist", + client.pathExists(backupPath + "level1/level2/level3/deep-file.txt")); + } + + @Test + public void testBackupRestore() throws Exception { + String backupPath = "backup-restore-test/"; + String restorePath = "restore-test/"; + + // Create backup + String originalContent = "Original backup content"; + pushContent(backupPath + "backup-file.txt", originalContent); + + // Simulate restore by copying content + try (var input = client.pullStream(backupPath + "backup-file.txt"); + var output = client.pushStream(restorePath + "restored-file.txt")) { + + byte[] buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = input.read(buffer)) != -1) { + output.write(buffer, 0, bytesRead); + } + } + + // Verify restore + assertTrue("Restored file should exist", client.pathExists(restorePath + "restored-file.txt")); + + // Verify content + try (var input = client.pullStream(restorePath + "restored-file.txt")) { + byte[] buffer = new byte[1024]; + int bytesRead = input.read(buffer); + String restoredContent = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8); + assertEquals("Restored content should match", originalContent, restoredContent); + } + } + + @Test + public void testBackupWithLargeFiles() throws Exception { + String backupPath = "large-file-backup-test/"; + + // Create large file + StringBuilder contentBuilder = new StringBuilder(); + for (int i = 0; i < 10000; i++) { + contentBuilder.append("This is line ").append(i).append(" of the large backup file.\n"); + } + String largeContent = contentBuilder.toString(); + + pushContent(backupPath + "large-backup.txt", largeContent); + + // Verify large file + assertTrue( + "Large backup file should exist", client.pathExists(backupPath + "large-backup.txt")); + assertEquals( + "Large file length should match", + largeContent.length(), + client.length(backupPath + "large-backup.txt")); + } + + @Test + public void testBackupWithBinaryFiles() throws Exception { + String backupPath = "binary-backup-test/"; + + // Create binary file + byte[] binaryData = new byte[1024]; + for (int i = 0; i < binaryData.length; i++) { + binaryData[i] = (byte) (i % 256); + } + + pushContent(backupPath + "binary-backup.bin", binaryData); + + // Verify binary file + assertTrue( + "Binary backup file should exist", client.pathExists(backupPath + "binary-backup.bin")); + assertEquals( + "Binary file length should match", + binaryData.length, + client.length(backupPath + "binary-backup.bin")); + } + + @Test + public void testBackupCleanup() throws Exception { + String backupPath = "backup-cleanup-test/"; + + // Create multiple backups + for (int i = 1; i <= 5; i++) { + pushContent(backupPath + "backup" + i + "/backup-file.txt", "Backup " + i + " content"); + } + + // Verify all backups exist + for (int i = 1; i <= 5; i++) { + assertTrue( + "Backup " + i + " should exist", client.pathExists(backupPath + "backup" + i + "/")); + } + + // Cleanup old backups (keep only last 3) + for (int i = 1; i <= 2; i++) { + client.deleteDirectory(backupPath + "backup" + i + "/"); + } + + // Verify cleanup + for (int i = 1; i <= 2; i++) { + assertFalse( + "Old backup " + i + " should not exist", + client.pathExists(backupPath + "backup" + i + "/")); + } + for (int i = 3; i <= 5; i++) { + assertTrue( + "Recent backup " + i + " should exist", + client.pathExists(backupPath + "backup" + i + "/")); + } + } + + @Test + public void testBackupWithMetadata() throws Exception { + String backupPath = "metadata-backup-test/"; + + // Create backup with metadata files + pushContent( + backupPath + "backup-metadata.json", + "{\"timestamp\":\"2023-01-01T00:00:00Z\",\"version\":\"1.0\"}"); + pushContent(backupPath + "backup-data.txt", "Backup data content"); + + // Verify metadata files + assertTrue( + "Metadata file should exist", client.pathExists(backupPath + "backup-metadata.json")); + assertTrue("Data file should exist", client.pathExists(backupPath + "backup-data.txt")); + } + + @Test + public void testConcurrentBackups() throws Exception { + String backupPath = "concurrent-backup-test/"; + + // Simulate concurrent backups + String[] backupNames = {"backup1", "backup2", "backup3"}; + String[] contents = {"Content 1", "Content 2", "Content 3"}; + + // Create backups concurrently (simulated) + for (int i = 0; i < backupNames.length; i++) { + pushContent(backupPath + backupNames[i] + "/backup-file.txt", contents[i]); + } + + // Verify all backups exist + for (String backupName : backupNames) { + assertTrue( + "Backup should exist: " + backupName, client.pathExists(backupPath + backupName + "/")); + } + } + + private void createBackup(String backupPath, String content) throws BlobException { + client.createDirectory(backupPath); + pushContent(backupPath + "backup-file.txt", content); + } +} diff --git a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobIndexInputTest.java b/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobIndexInputTest.java new file mode 100644 index 000000000000..ccd681eab70d --- /dev/null +++ b/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobIndexInputTest.java @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.blob; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import org.junit.Test; + +public class BlobIndexInputTest extends AbstractBlobClientTest { + + @Test + public void testBasicIndexInput() throws Exception { + String path = "index-input-test.txt"; + String content = "Index input test content"; + + // Write content + pushContent(path, content); + + // Read using BlobIndexInput + try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + byte[] buffer = new byte[1024]; + input.readBytes(buffer, 0, content.length()); + String readContent = new String(buffer, 0, content.length(), StandardCharsets.UTF_8); + assertEquals("Content should match", content, readContent); + } + } + + @Test + public void testIndexInputSeek() throws Exception { + String path = "index-input-seek-test.txt"; + String content = "Index input seek test content"; + + // Write content + pushContent(path, content); + + // Test seeking + try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + // Seek to middle of content + long seekPosition = content.length() / 2; + input.seek(seekPosition); + + // Read remaining content + byte[] buffer = new byte[1024]; + String expectedContent = content.substring((int) seekPosition); + input.readBytes(buffer, 0, expectedContent.length()); + String readContent = new String(buffer, 0, expectedContent.length(), StandardCharsets.UTF_8); + assertEquals("Content from seek position should match", expectedContent, readContent); + } + } + + @Test + public void testIndexInputLength() throws Exception { + String path = "index-input-length-test.txt"; + String content = "Length test content"; + + // Write content + pushContent(path, content); + + // Test length + try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + assertEquals("Length should match", content.length(), input.length()); + } + } + + @Test + public void testIndexInputReadByte() throws Exception { + String path = "index-input-byte-test.txt"; + String content = "Byte read test"; + + // Write content + pushContent(path, content); + + // Test reading byte by byte + try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + StringBuilder readContent = new StringBuilder(); + for (int i = 0; i < content.length(); i++) { + byte b = input.readByte(); + readContent.append((char) b); + } + assertEquals("Byte by byte content should match", content, readContent.toString()); + } + } + + @Test + public void testIndexInputReadBytes() throws Exception { + String path = "index-input-bytes-test.txt"; + String content = "Bytes read test content"; + + // Write content + pushContent(path, content); + + // Test reading bytes + try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + byte[] buffer = new byte[10]; + StringBuilder readContent = new StringBuilder(); + + // Read all content in chunks + long remaining = input.length(); + while (remaining > 0) { + int toRead = (int) Math.min(buffer.length, remaining); + input.readBytes(buffer, 0, toRead); + readContent.append(new String(buffer, 0, toRead, StandardCharsets.UTF_8)); + remaining -= toRead; + } + + assertEquals("Bytes content should match", content, readContent.toString()); + } + } + + @Test + public void testIndexInputSeekToEnd() throws Exception { + String path = "index-input-seek-end-test.txt"; + String content = "Seek to end test"; + + // Write content + pushContent(path, content); + + // Test seeking to end + try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + input.seek(content.length()); + + // Should be at end, no more bytes to read + try { + input.readByte(); + fail("Should throw EOFException when reading past end"); + } catch (IOException e) { + // Expected + } + } + } + + @Test + public void testIndexInputSeekBeyondEnd() throws Exception { + String path = "index-input-seek-beyond-test.txt"; + String content = "Seek beyond end test"; + + // Write content + pushContent(path, content); + + // Test seeking beyond end + try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + try { + input.seek(content.length() + 1); + fail("Should throw IOException when seeking beyond end"); + } catch (IOException e) { + // Expected + } + } + } + + @Test + public void testIndexInputGetFilePointer() throws Exception { + String path = "index-input-pointer-test.txt"; + String content = "File pointer test content"; + + // Write content + pushContent(path, content); + + // Test file pointer + try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + assertEquals("Initial position should be 0", 0, input.getFilePointer()); + + // Read some bytes + byte[] buffer = new byte[5]; + input.readBytes(buffer, 0, buffer.length); + assertEquals("Position should be 5 after reading 5 bytes", 5, input.getFilePointer()); + + // Seek to different position + input.seek(10); + assertEquals("Position should be 10 after seek", 10, input.getFilePointer()); + } + } + + @Test + public void testIndexInputLargeFile() throws Exception { + String path = "index-input-large-test.txt"; + StringBuilder contentBuilder = new StringBuilder(); + + // Create large content (1MB) + for (int i = 0; i < 10000; i++) { + contentBuilder.append("This is line ").append(i).append(" of the large file.\n"); + } + String content = contentBuilder.toString(); + + // Write content + pushContent(path, content); + + // Test reading large file + try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + assertEquals("Length should match", content.length(), input.length()); + + // Read in chunks + byte[] buffer = new byte[8192]; + StringBuilder readContent = new StringBuilder(); + + // Read all content in chunks + long remaining = input.length(); + while (remaining > 0) { + int toRead = (int) Math.min(buffer.length, remaining); + input.readBytes(buffer, 0, toRead); + readContent.append(new String(buffer, 0, toRead, StandardCharsets.UTF_8)); + remaining -= toRead; + } + + assertEquals("Large content should match", content, readContent.toString()); + } + } + + @Test + public void testIndexInputEmptyFile() throws Exception { + String path = "index-input-empty-test.txt"; + String content = ""; + + // Write empty content + pushContent(path, content); + + // Test reading empty file + try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + assertEquals("Length should be 0", 0, input.length()); + assertEquals("Position should be 0", 0, input.getFilePointer()); + + // Should be at end immediately + try { + input.readByte(); + fail("Should throw EOFException when reading from empty file"); + } catch (IOException e) { + // Expected + } + } + } + + @Test + public void testIndexInputClose() throws Exception { + String path = "index-input-close-test.txt"; + String content = "Close test content"; + + // Write content + pushContent(path, content); + + // Test closing + BlobIndexInput input = new BlobIndexInput(path, client, client.length(path)); + input.close(); + + // Test that operations on closed input throw exception + try { + input.readByte(); + fail("Should throw IOException when reading from closed input"); + } catch (IOException e) { + // Expected + } + + try { + input.seek(0); + fail("Should throw IOException when seeking on closed input"); + } catch (IOException e) { + // Expected + } + } + + @Test + public void testIndexInputMultipleClose() throws Exception { + String path = "index-input-multiple-close-test.txt"; + String content = "Multiple close test content"; + + // Write content + pushContent(path, content); + + // Test multiple close calls + BlobIndexInput input = new BlobIndexInput(path, client, client.length(path)); + input.close(); + input.close(); // Should not throw exception + } +} diff --git a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobInstallShardTest.java b/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobInstallShardTest.java new file mode 100644 index 000000000000..e89ca6e2a402 --- /dev/null +++ b/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobInstallShardTest.java @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.blob; + +import java.nio.charset.StandardCharsets; +import org.junit.Test; + +public class BlobInstallShardTest extends AbstractBlobClientTest { + + @Test + public void testInstallShard() throws Exception { + String shardPath = "install-shard-test/"; + + // Create shard structure + client.createDirectory(shardPath); + client.createDirectory(shardPath + "index/"); + client.createDirectory(shardPath + "conf/"); + + // Add shard files + pushContent(shardPath + "index/segments_1", "Shard index segments"); + pushContent(shardPath + "index/_0.cfs", "Shard index file"); + pushContent(shardPath + "conf/solrconfig.xml", "Shard configuration"); + pushContent(shardPath + "conf/schema.xml", "Shard schema"); + + // Verify shard structure + assertTrue("Shard directory should exist", client.pathExists(shardPath)); + assertTrue("Index directory should exist", client.pathExists(shardPath + "index/")); + assertTrue("Conf directory should exist", client.pathExists(shardPath + "conf/")); + assertTrue("Segments file should exist", client.pathExists(shardPath + "index/segments_1")); + assertTrue("Index file should exist", client.pathExists(shardPath + "index/_0.cfs")); + assertTrue("Config file should exist", client.pathExists(shardPath + "conf/solrconfig.xml")); + assertTrue("Schema file should exist", client.pathExists(shardPath + "conf/schema.xml")); + } + + @Test + public void testInstallShardWithMultipleIndexFiles() throws Exception { + String shardPath = "multi-index-shard-test/"; + + // Create shard structure + client.createDirectory(shardPath); + client.createDirectory(shardPath + "index/"); + + // Add multiple index files + String[] indexFiles = {"segments_1", "_0.cfs", "_0.cfe", "_0.si", "_1.cfs", "_1.cfe", "_1.si"}; + + for (String indexFile : indexFiles) { + pushContent(shardPath + "index/" + indexFile, "Index file content: " + indexFile); + } + + // Verify all index files exist + for (String indexFile : indexFiles) { + assertTrue( + "Index file should exist: " + indexFile, + client.pathExists(shardPath + "index/" + indexFile)); + } + } + + @Test + public void testInstallShardWithDataFiles() throws Exception { + String shardPath = "data-shard-test/"; + + // Create shard structure + client.createDirectory(shardPath); + client.createDirectory(shardPath + "data/"); + + // Add data files + String[] dataFiles = { + "tlog.0000000000000000001", "tlog.0000000000000000002", "tlog.0000000000000000003" + }; + + for (String dataFile : dataFiles) { + pushContent(shardPath + "data/" + dataFile, "Transaction log: " + dataFile); + } + + // Verify all data files exist + for (String dataFile : dataFiles) { + assertTrue( + "Data file should exist: " + dataFile, client.pathExists(shardPath + "data/" + dataFile)); + } + } + + @Test + public void testInstallShardWithConfiguration() throws Exception { + String shardPath = "config-shard-test/"; + + // Create shard structure + client.createDirectory(shardPath); + client.createDirectory(shardPath + "conf/"); + + // Add configuration files + String solrConfig = + "\n" + + "\n" + + " LATEST\n" + + " \n" + + ""; + + String schema = + "\n" + + "\n" + + " \n" + + ""; + + pushContent(shardPath + "conf/solrconfig.xml", solrConfig); + pushContent(shardPath + "conf/schema.xml", schema); + + // Verify configuration files + assertTrue("Solr config should exist", client.pathExists(shardPath + "conf/solrconfig.xml")); + assertTrue("Schema should exist", client.pathExists(shardPath + "conf/schema.xml")); + + // Verify content + try (var input = client.pullStream(shardPath + "conf/solrconfig.xml")) { + byte[] buffer = new byte[1024]; + int bytesRead = input.read(buffer); + String readContent = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8); + assertTrue( + "Solr config should contain expected content", + readContent.contains("luceneMatchVersion")); + } + } + + @Test + public void testInstallShardWithLargeIndex() throws Exception { + String shardPath = "large-index-shard-test/"; + + // Create shard structure + client.createDirectory(shardPath); + client.createDirectory(shardPath + "index/"); + + // Create large index file + StringBuilder largeContent = new StringBuilder(); + for (int i = 0; i < 50000; i++) { + largeContent.append("Index data line ").append(i).append("\n"); + } + + pushContent(shardPath + "index/large-index.cfs", largeContent.toString()); + + // Verify large index file + assertTrue( + "Large index file should exist", client.pathExists(shardPath + "index/large-index.cfs")); + assertEquals( + "Large index file length should match", + largeContent.length(), + client.length(shardPath + "index/large-index.cfs")); + } + + @Test + public void testInstallShardWithBinaryIndex() throws Exception { + String shardPath = "binary-index-shard-test/"; + + // Create shard structure + client.createDirectory(shardPath); + client.createDirectory(shardPath + "index/"); + + // Create binary index file + byte[] binaryData = new byte[2048]; + for (int i = 0; i < binaryData.length; i++) { + binaryData[i] = (byte) (i % 256); + } + + pushContent(shardPath + "index/binary-index.cfs", binaryData); + + // Verify binary index file + assertTrue( + "Binary index file should exist", client.pathExists(shardPath + "index/binary-index.cfs")); + assertEquals( + "Binary index file length should match", + binaryData.length, + client.length(shardPath + "index/binary-index.cfs")); + } + + @Test + public void testInstallShardWithNestedStructure() throws Exception { + String shardPath = "nested-shard-test/"; + + // Create nested shard structure + client.createDirectory(shardPath); + client.createDirectory(shardPath + "index/"); + client.createDirectory(shardPath + "conf/"); + client.createDirectory(shardPath + "data/"); + client.createDirectory(shardPath + "logs/"); + + // Add files at different levels + pushContent(shardPath + "index/segments_1", "Segments file"); + pushContent(shardPath + "conf/solrconfig.xml", "Config file"); + pushContent(shardPath + "data/tlog.1", "Transaction log"); + pushContent(shardPath + "logs/solr.log", "Log file"); + + // Verify nested structure + assertTrue("Root shard should exist", client.pathExists(shardPath)); + assertTrue("Index directory should exist", client.pathExists(shardPath + "index/")); + assertTrue("Conf directory should exist", client.pathExists(shardPath + "conf/")); + assertTrue("Data directory should exist", client.pathExists(shardPath + "data/")); + assertTrue("Logs directory should exist", client.pathExists(shardPath + "logs/")); + + // Verify files exist + assertTrue("Segments file should exist", client.pathExists(shardPath + "index/segments_1")); + assertTrue("Config file should exist", client.pathExists(shardPath + "conf/solrconfig.xml")); + assertTrue("Transaction log should exist", client.pathExists(shardPath + "data/tlog.1")); + assertTrue("Log file should exist", client.pathExists(shardPath + "logs/solr.log")); + } + + @Test + public void testInstallShardWithMetadata() throws Exception { + String shardPath = "metadata-shard-test/"; + + // Create shard structure + client.createDirectory(shardPath); + + // Add metadata files + String metadata = + "{\n" + + " \"shardId\": \"shard1\",\n" + + " \"coreName\": \"test-core\",\n" + + " \"version\": \"1.0\",\n" + + " \"timestamp\": \"2023-01-01T00:00:00Z\"\n" + + "}"; + + pushContent(shardPath + "shard-metadata.json", metadata); + pushContent(shardPath + "index/segments_1", "Index segments"); + + // Verify metadata + assertTrue("Metadata file should exist", client.pathExists(shardPath + "shard-metadata.json")); + assertTrue("Index file should exist", client.pathExists(shardPath + "index/segments_1")); + + // Verify metadata content + try (var input = client.pullStream(shardPath + "shard-metadata.json")) { + byte[] buffer = new byte[1024]; + int bytesRead = input.read(buffer); + String readContent = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8); + assertTrue("Metadata should contain shard ID", readContent.contains("shard1")); + assertTrue("Metadata should contain core name", readContent.contains("test-core")); + } + } + + @Test + public void testInstallShardCleanup() throws Exception { + String shardPath = "cleanup-shard-test/"; + + // Create shard structure + client.createDirectory(shardPath); + client.createDirectory(shardPath + "index/"); + client.createDirectory(shardPath + "conf/"); + + // Add shard files + pushContent(shardPath + "index/segments_1", "Index segments"); + pushContent(shardPath + "conf/solrconfig.xml", "Config file"); + + // Verify shard exists + assertTrue("Shard should exist", client.pathExists(shardPath)); + + // Cleanup shard + client.deleteDirectory(shardPath); + + // Verify shard is cleaned up + assertFalse("Shard should not exist after cleanup", client.pathExists(shardPath)); + assertFalse( + "Index directory should not exist after cleanup", client.pathExists(shardPath + "index/")); + assertFalse( + "Conf directory should not exist after cleanup", client.pathExists(shardPath + "conf/")); + } +} diff --git a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobOutputStreamTest.java b/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobOutputStreamTest.java new file mode 100644 index 000000000000..f943264fa410 --- /dev/null +++ b/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobOutputStreamTest.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.blob; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import org.junit.Test; + +public class BlobOutputStreamTest extends AbstractBlobClientTest { + + @Test + public void testBasicOutputStream() throws Exception { + String path = "output-stream-test.txt"; + String content = "Output stream test content"; + + try (OutputStream output = client.pushStream(path)) { + output.write(content.getBytes(StandardCharsets.UTF_8)); + } + + // Verify content was written + assertTrue("File should exist", client.pathExists(path)); + + try (InputStream input = client.pullStream(path)) { + byte[] buffer = new byte[1024]; + int bytesRead = input.read(buffer); + String readContent = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8); + assertEquals("Content should match", content, readContent); + } + } + + @Test + public void testOutputStreamWriteByte() throws Exception { + String path = "output-stream-byte-test.txt"; + String content = "Byte by byte write test"; + + try (OutputStream output = client.pushStream(path)) { + for (byte b : content.getBytes(StandardCharsets.UTF_8)) { + output.write(b); + } + } + + // Verify content was written + assertTrue("File should exist", client.pathExists(path)); + + try (InputStream input = client.pullStream(path)) { + byte[] buffer = new byte[1024]; + int bytesRead = input.read(buffer); + String readContent = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8); + assertEquals("Content should match", content, readContent); + } + } + + @Test + public void testOutputStreamWriteByteArray() throws Exception { + String path = "output-stream-array-test.txt"; + String content = "Byte array write test"; + byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); + + try (OutputStream output = client.pushStream(path)) { + output.write(contentBytes); + } + + // Verify content was written + assertTrue("File should exist", client.pathExists(path)); + + try (InputStream input = client.pullStream(path)) { + byte[] buffer = new byte[1024]; + int bytesRead = input.read(buffer); + String readContent = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8); + assertEquals("Content should match", content, readContent); + } + } + + @Test + public void testOutputStreamWriteByteArrayWithOffset() throws Exception { + String path = "output-stream-offset-test.txt"; + String fullContent = "Full content for offset test"; + String partialContent = "offset test"; // Last part + byte[] fullBytes = fullContent.getBytes(StandardCharsets.UTF_8); + int offset = fullContent.indexOf(partialContent); + + try (OutputStream output = client.pushStream(path)) { + output.write(fullBytes, offset, partialContent.length()); + } + + // Verify content was written + assertTrue("File should exist", client.pathExists(path)); + + try (InputStream input = client.pullStream(path)) { + byte[] buffer = new byte[1024]; + int bytesRead = input.read(buffer); + String readContent = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8); + assertEquals("Content should match", partialContent, readContent); + } + } + + @Test + public void testOutputStreamFlush() throws Exception { + String path = "output-stream-flush-test.txt"; + String content = "Flush test content"; + + try (OutputStream output = client.pushStream(path)) { + output.write(content.getBytes(StandardCharsets.UTF_8)); + output.flush(); + + // Verify content is available after flush + assertTrue("File should exist after flush", client.pathExists(path)); + } + } + + @Test + public void testOutputStreamClose() throws Exception { + String path = "output-stream-close-test.txt"; + String content = "Close test content"; + + OutputStream output = client.pushStream(path); + output.write(content.getBytes(StandardCharsets.UTF_8)); + output.close(); + + // Verify content was written + assertTrue("File should exist after close", client.pathExists(path)); + + // Test that operations on closed stream throw exception + try { + output.write(1); + fail("Should throw IOException when writing to closed stream"); + } catch (IOException e) { + // Expected + } + + try { + output.flush(); + fail("Should throw IOException when flushing closed stream"); + } catch (IOException e) { + // Expected + } + } + + @Test + public void testOutputStreamMultipleClose() throws Exception { + String path = "output-stream-multiple-close-test.txt"; + String content = "Multiple close test content"; + + OutputStream output = client.pushStream(path); + output.write(content.getBytes(StandardCharsets.UTF_8)); + output.close(); + output.close(); // Should not throw exception + + // Verify content was written + assertTrue("File should exist", client.pathExists(path)); + } + + @Test + public void testOutputStreamLargeData() throws Exception { + String path = "output-stream-large-test.txt"; + StringBuilder contentBuilder = new StringBuilder(); + + // Create large content (2MB) + for (int i = 0; i < 20000; i++) { + contentBuilder.append("This is line ").append(i).append(" of the large file.\n"); + } + String content = contentBuilder.toString(); + + try (OutputStream output = client.pushStream(path)) { + output.write(content.getBytes(StandardCharsets.UTF_8)); + } + + // Verify content was written + assertTrue("Large file should exist", client.pathExists(path)); + assertEquals("File length should match", content.length(), client.length(path)); + + // Verify content integrity + try (InputStream input = client.pullStream(path)) { + byte[] buffer = new byte[8192]; + StringBuilder readContentBuilder = new StringBuilder(); + int bytesRead; + while ((bytesRead = input.read(buffer)) != -1) { + readContentBuilder.append(new String(buffer, 0, bytesRead, StandardCharsets.UTF_8)); + } + assertEquals("Large content should match", content, readContentBuilder.toString()); + } + } + + @Test + public void testOutputStreamChunkedWrite() throws Exception { + String path = "output-stream-chunked-test.txt"; + String content = "Chunked write test content"; + byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); + + try (OutputStream output = client.pushStream(path)) { + // Write in small chunks + int chunkSize = 5; + for (int i = 0; i < contentBytes.length; i += chunkSize) { + int remaining = Math.min(chunkSize, contentBytes.length - i); + output.write(contentBytes, i, remaining); + } + } + + // Verify content was written correctly + assertTrue("File should exist", client.pathExists(path)); + + try (InputStream input = client.pullStream(path)) { + byte[] buffer = new byte[1024]; + int bytesRead = input.read(buffer); + String readContent = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8); + assertEquals("Chunked content should match", content, readContent); + } + } + + @Test + public void testOutputStreamBinaryData() throws Exception { + String path = "output-stream-binary-test.bin"; + byte[] binaryData = new byte[1024]; + + // Fill with some binary data + for (int i = 0; i < binaryData.length; i++) { + binaryData[i] = (byte) (i % 256); + } + + try (OutputStream output = client.pushStream(path)) { + output.write(binaryData); + } + + // Verify binary data was written + assertTrue("Binary file should exist", client.pathExists(path)); + assertEquals("Binary file length should match", binaryData.length, client.length(path)); + + // Verify binary data integrity + try (InputStream input = client.pullStream(path)) { + byte[] readData = new byte[binaryData.length]; + int bytesRead = input.read(readData); + assertEquals("Should read all bytes", binaryData.length, bytesRead); + + for (int i = 0; i < binaryData.length; i++) { + assertEquals("Binary data should match at position " + i, binaryData[i], readData[i]); + } + } + } +} diff --git a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobPathsTest.java b/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobPathsTest.java new file mode 100644 index 000000000000..7cd3606d7d6f --- /dev/null +++ b/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobPathsTest.java @@ -0,0 +1,332 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.blob; + +import org.junit.Test; + +public class BlobPathsTest extends AbstractBlobClientTest { + + @Test + public void testPathExists() throws Exception { + String path = "path-exists-test-" + java.util.UUID.randomUUID() + ".txt"; + + // Initially should not exist + assertFalse("Path should not exist initially", client.pathExists(path)); + + // Create file + pushContent(path, "test content"); + + // Should exist now + assertTrue("Path should exist after creation", client.pathExists(path)); + } + + @Test + public void testDirectoryExists() throws Exception { + String dirPath = "test-directory-" + java.util.UUID.randomUUID() + "/"; + + // Initially should not exist + assertFalse("Directory should not exist initially", client.pathExists(dirPath)); + + // Create directory + client.createDirectory(dirPath); + + // Should exist now + assertTrue("Directory should exist after creation", client.pathExists(dirPath)); + } + + @Test + public void testIsDirectory() throws Exception { + String dirPath = "is-directory-test/"; + String filePath = "is-directory-test.txt"; + + // Create directory + client.createDirectory(dirPath); + assertTrue("Should be a directory", client.isDirectory(dirPath)); + + // Create file + pushContent(filePath, "test content"); + assertFalse("Should not be a directory", client.isDirectory(filePath)); + } + + @Test + public void testFileLength() throws Exception { + String path = "file-length-test.txt"; + String content = "File length test content"; + + // Create file + pushContent(path, content); + + // Check length + assertEquals("File length should match", content.length(), client.length(path)); + } + + @Test + public void testDirectoryLength() throws Exception { + String dirPath = "directory-length-test/"; + + // Create directory + client.createDirectory(dirPath); + + // Should throw exception when getting length of directory + try { + client.length(dirPath); + fail("Should throw exception when getting length of directory"); + } catch (BlobException e) { + // Expected + } + } + + @Test + public void testListDirectory() throws Exception { + String dirPath = "list-directory-test/"; + + // Create directory + client.createDirectory(dirPath); + + // Initially should be empty + String[] files = client.listDir(dirPath); + assertEquals("Directory should be empty initially", 0, files.length); + + // Add some files + String[] fileNames = {"file1.txt", "file2.txt", "subdir/"}; + for (String fileName : fileNames) { + String fullPath = dirPath + fileName; + if (fileName.endsWith("/")) { + client.createDirectory(fullPath); + } else { + pushContent(fullPath, "Content of " + fileName); + } + } + + // List directory contents + files = client.listDir(dirPath); + assertEquals("Should list all files and directories", fileNames.length, files.length); + + // Verify all files are listed + for (String fileName : fileNames) { + boolean found = false; + for (String listedFile : files) { + if (fileName.equals(listedFile)) { + found = true; + break; + } + } + assertTrue("Should find file: " + fileName, found); + } + } + + @Test + public void testListAll() throws Exception { + String dirPath = "list-all-test/"; + + // Create directory structure + client.createDirectory(dirPath); + client.createDirectory(dirPath + "subdir1/"); + client.createDirectory(dirPath + "subdir2/"); + + pushContent(dirPath + "file1.txt", "Content 1"); + pushContent(dirPath + "file2.txt", "Content 2"); + pushContent(dirPath + "subdir1/file3.txt", "Content 3"); + pushContent(dirPath + "subdir2/file4.txt", "Content 4"); + + // List all files recursively + java.util.Set allFiles = new java.util.HashSet<>(); + listAllRecursive(dirPath, allFiles); + + // Should find all files + assertTrue("Should find file1.txt", allFiles.contains(dirPath + "file1.txt")); + assertTrue("Should find file2.txt", allFiles.contains(dirPath + "file2.txt")); + assertTrue("Should find subdir1/file3.txt", allFiles.contains(dirPath + "subdir1/file3.txt")); + assertTrue("Should find subdir2/file4.txt", allFiles.contains(dirPath + "subdir2/file4.txt")); + } + + private void listAllRecursive(String dirPath, java.util.Set allFiles) + throws BlobException { + String[] files = client.listDir(dirPath); + for (String file : files) { + String fullPath = dirPath + file; + if (file.endsWith("/")) { + // It's a directory + allFiles.add(fullPath); + listAllRecursive(fullPath, allFiles); + } else { + // It's a file + allFiles.add(fullPath); + } + } + } + + @Test + public void testDeleteFile() throws Exception { + String path = "delete-file-test.txt"; + + // Create file + pushContent(path, "test content"); + assertTrue("File should exist", client.pathExists(path)); + + // Delete file + client.delete(java.util.Set.of(path)); + + // Should not exist anymore + assertFalse("File should not exist after deletion", client.pathExists(path)); + } + + @Test + public void testDeleteDirectory() throws Exception { + String dirPath = "delete-directory-test/"; + String filePath = dirPath + "nested-file.txt"; + + // Create directory and file + client.createDirectory(dirPath); + pushContent(filePath, "nested content"); + + assertTrue("Directory should exist", client.pathExists(dirPath)); + assertTrue("File should exist", client.pathExists(filePath)); + + // Delete directory + client.deleteDirectory(dirPath); + + // Should not exist anymore + assertFalse("Directory should not exist after deletion", client.pathExists(dirPath)); + assertFalse("File should not exist after deletion", client.pathExists(filePath)); + } + + @Test + public void testDeleteNonExistentFile() throws Exception { + String path = "non-existent-file.txt"; + + // Should not exist + assertFalse("File should not exist", client.pathExists(path)); + + // Delete non-existent file should not throw exception + client.delete(java.util.Set.of(path)); + } + + @Test + public void testDeleteNonExistentDirectory() throws Exception { + String dirPath = "non-existent-directory/"; + + // Should not exist + assertFalse("Directory should not exist", client.pathExists(dirPath)); + + // Delete non-existent directory should not throw exception + client.deleteDirectory(dirPath); + } + + @Test + public void testNestedDirectories() throws Exception { + String rootDir = "nested-test/"; + String subDir1 = rootDir + "subdir1/"; + String subDir2 = rootDir + "subdir2/"; + String deepDir = subDir1 + "deepdir/"; + + // Create nested directory structure + client.createDirectory(rootDir); + client.createDirectory(subDir1); + client.createDirectory(subDir2); + client.createDirectory(deepDir); + + // Verify all directories exist + assertTrue("Root directory should exist", client.pathExists(rootDir)); + assertTrue("Sub directory 1 should exist", client.pathExists(subDir1)); + assertTrue("Sub directory 2 should exist", client.pathExists(subDir2)); + assertTrue("Deep directory should exist", client.pathExists(deepDir)); + + // Add files to different levels + pushContent(rootDir + "root-file.txt", "Root file content"); + pushContent(subDir1 + "sub-file.txt", "Sub file content"); + pushContent(deepDir + "deep-file.txt", "Deep file content"); + + // Verify files exist + assertTrue("Root file should exist", client.pathExists(rootDir + "root-file.txt")); + assertTrue("Sub file should exist", client.pathExists(subDir1 + "sub-file.txt")); + assertTrue("Deep file should exist", client.pathExists(deepDir + "deep-file.txt")); + } + + @Test + public void testPathSanitization() throws Exception { + // Test various path formats + String[] testPaths = { + "simple-file.txt", + "/leading-slash.txt", + "trailing-slash/", + "/both-slashes/", + "nested/path/file.txt", + "//double-slash.txt", + " spaced-file.txt ", + "special-chars!@#$%^&*().txt" + }; + + for (String testPath : testPaths) { + try { + String sanitizedPath = client.sanitizedPath(testPath); + assertNotNull("Sanitized path should not be null", sanitizedPath); + assertFalse("Sanitized path should not start with slash", sanitizedPath.startsWith("/")); + } catch (BlobException e) { + // Some paths might be invalid, which is expected + } + } + } + + @Test + public void testFilePathSanitization() throws Exception { + // Test file path sanitization + String[] validFilePaths = { + "simple-file.txt", "nested/path/file.txt", "file-with-dashes.txt", "file_with_underscores.txt" + }; + + for (String filePath : validFilePaths) { + try { + String sanitizedPath = client.sanitizedFilePath(filePath); + assertNotNull("Sanitized file path should not be null", sanitizedPath); + assertFalse("Sanitized file path should not end with slash", sanitizedPath.endsWith("/")); + } catch (BlobException e) { + fail("Valid file path should not throw exception: " + filePath); + } + } + + // Test invalid file paths + String[] invalidFilePaths = {"file-with-trailing-slash/", "", " "}; + + for (String filePath : invalidFilePaths) { + try { + client.sanitizedFilePath(filePath); + fail("Invalid file path should throw exception: " + filePath); + } catch (BlobException e) { + // Expected + } + } + } + + @Test + public void testDirectoryPathSanitization() throws Exception { + // Test directory path sanitization + String[] testDirPaths = { + "simple-dir", "nested/path/dir", "dir-with-dashes", "dir_with_underscores" + }; + + for (String dirPath : testDirPaths) { + try { + String sanitizedPath = client.sanitizedDirPath(dirPath); + assertNotNull("Sanitized directory path should not be null", sanitizedPath); + assertTrue("Sanitized directory path should end with slash", sanitizedPath.endsWith("/")); + } catch (BlobException e) { + fail("Valid directory path should not throw exception: " + dirPath); + } + } + } +} diff --git a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobReadWriteTest.java b/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobReadWriteTest.java new file mode 100644 index 000000000000..93256ae7058c --- /dev/null +++ b/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobReadWriteTest.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.blob; + +import com.carrotsearch.randomizedtesting.generators.RandomBytes; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import org.junit.Test; + +public class BlobReadWriteTest extends AbstractBlobClientTest { + + @Test + public void testBasicReadWrite() throws Exception { + String path = "test-file.txt"; + String content = "Hello, Azure Blob Storage!"; + + // Write content + pushContent(path, content); + + // Read content + try (InputStream input = client.pullStream(path)) { + byte[] buffer = new byte[1024]; + int bytesRead = input.read(buffer); + String readContent = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8); + assertEquals("Content should match", content, readContent); + } + } + + @Test + public void testLargeFileReadWrite() throws Exception { + String path = "large-file.txt"; + StringBuilder contentBuilder = new StringBuilder(); + + // Create a large content (1MB) + for (int i = 0; i < 10000; i++) { + contentBuilder.append("This is line ").append(i).append(" of the large file.\n"); + } + String content = contentBuilder.toString(); + + // Write content + pushContent(path, content); + + // Verify file exists and has correct length + assertTrue("File should exist", client.pathExists(path)); + assertEquals("File length should match", content.length(), client.length(path)); + + // Read content back + try (InputStream input = client.pullStream(path)) { + byte[] buffer = new byte[8192]; + StringBuilder readContentBuilder = new StringBuilder(); + int bytesRead; + while ((bytesRead = input.read(buffer)) != -1) { + readContentBuilder.append(new String(buffer, 0, bytesRead, StandardCharsets.UTF_8)); + } + assertEquals("Content should match", content, readContentBuilder.toString()); + } + } + + @Test + public void testBinaryDataReadWrite() throws Exception { + String path = "binary-file.bin"; + byte[] binaryData = new byte[1024]; + + // Fill with some binary data + for (int i = 0; i < binaryData.length; i++) { + binaryData[i] = (byte) (i % 256); + } + + // Write binary data + pushContent(path, binaryData); + + // Read binary data back + try (InputStream input = client.pullStream(path)) { + byte[] readData = new byte[binaryData.length]; + int bytesRead = input.read(readData); + assertEquals("Should read all bytes", binaryData.length, bytesRead); + + for (int i = 0; i < binaryData.length; i++) { + assertEquals("Binary data should match at position " + i, binaryData[i], readData[i]); + } + } + } + + @Test + public void testConcurrentReadWrite() throws Exception { + String path = "concurrent-file.txt"; + String content = "Concurrent read/write test content"; + + // Write content + pushContent(path, content); + + // Read from multiple streams concurrently + try (InputStream input1 = client.pullStream(path); + InputStream input2 = client.pullStream(path)) { + + byte[] buffer1 = new byte[1024]; + byte[] buffer2 = new byte[1024]; + + int bytesRead1 = input1.read(buffer1); + int bytesRead2 = input2.read(buffer2); + + String readContent1 = new String(buffer1, 0, bytesRead1, StandardCharsets.UTF_8); + String readContent2 = new String(buffer2, 0, bytesRead2, StandardCharsets.UTF_8); + + assertEquals("Both reads should get same content", readContent1, readContent2); + assertEquals("Content should match original", content, readContent1); + } + } + + @Test + public void testStreamClose() throws Exception { + String path = "stream-close-test.txt"; + String content = "Stream close test content"; + + // Write content + pushContent(path, content); + + // Test that stream can be closed multiple times without exception + InputStream input = client.pullStream(path); + input.close(); + input.close(); // Should not throw exception + + // ResumableInputStream automatically resumes after close, so we can still read + // This tests the resumable behavior - a new stream is created on read + int firstByte = input.read(); + assertTrue( + "Stream should be resumable after close (got byte: " + firstByte + ")", + firstByte >= 0 || firstByte == -1); // Either valid byte or EOF + + // Close again after successful resume + input.close(); + } + + @Test + public void testEmptyFileReadWrite() throws Exception { + String path = "empty-file.txt"; + String content = ""; + + // Write empty content + pushContent(path, content); + + // Verify file exists + assertTrue("Empty file should exist", client.pathExists(path)); + assertEquals("Empty file should have zero length", 0, client.length(path)); + + // Read empty content + try (InputStream input = client.pullStream(path)) { + int bytesRead = input.read(); + assertEquals("Should return -1 for empty file", -1, bytesRead); + } + } + + @Test + public void testUnicodeContentReadWrite() throws Exception { + String path = "unicode-file.txt"; + String content = "Hello 世界! 🌍 Unicode test: αβγδε"; + + // Write Unicode content + pushContent(path, content); + + // Read Unicode content back + try (InputStream input = client.pullStream(path)) { + byte[] buffer = new byte[1024]; + int bytesRead = input.read(buffer); + String readContent = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8); + assertEquals("Unicode content should match", content, readContent); + } + } + + @Test + public void testOutputStreamFlush() throws Exception { + String path = "flush-test.txt"; + String content = "Flush test content"; + + // Write content with explicit flush + try (OutputStream output = client.pushStream(path)) { + output.write(content.getBytes(StandardCharsets.UTF_8)); + output.flush(); + } + + // Verify content was written + assertTrue("File should exist after flush", client.pathExists(path)); + + try (InputStream input = client.pullStream(path)) { + byte[] buffer = new byte[1024]; + int bytesRead = input.read(buffer); + String readContent = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8); + assertEquals("Content should match after flush", content, readContent); + } + } + + @Test + public void testReadWithConnectionLoss() throws Exception { + String key = "flush-very-large"; + + int numBytes = 2_000_000; // keep this small to avoid long retries with Azure client + pushContent(key, RandomBytes.randomBytesOfLength(random(), numBytes)); + + int numExceptions = 5; // fewer induced failures for Azure path + int bytesPerException = numBytes / numExceptions; + // Check we can re-read same content + + int maxBuffer = 100; + byte[] buffer = new byte[maxBuffer]; + boolean done = false; + try (InputStream input = client.pullStream(key)) { + long byteCount = 0; + long lastResetBucket = -1; + while (!done) { + // Use the same number of bytes no matter which method we are testing + int numBytesToRead = random().nextInt(maxBuffer) + 1; + // test both read() and read(buffer, off, len) + switch (random().nextInt(3)) { + // read() + case 0: + { + for (int i = 0; i < numBytesToRead && !done; i++) { + done = input.read() == -1; + if (!done) { + byteCount++; + } + } + } + break; + // read(byte, off, len) + case 1: + { + int readLen = input.read(buffer, 0, numBytesToRead); + if (readLen > 0) { + byteCount += readLen; + } else { + // We are done when readLen = -1 + done = true; + } + } + break; + // skip(len) + case 2: + { + // We only want to skip 1 because + long bytesSkipped = input.skip(numBytesToRead); + byteCount += bytesSkipped; + if (bytesSkipped < numBytesToRead) { + // We are done when no bytes are skipped + done = true; + } + } + break; + } + // Initiate a connection loss at the beginning of every "bytesPerException" cycle. + // The input stream will not immediately see an error, it will have pre-loaded some data. + long currentBucket = byteCount / bytesPerException; + if (currentBucket != lastResetBucket && (byteCount % bytesPerException <= maxBuffer)) { + try { + initiateBlobConnectionLoss(); + } catch (BlobException e) { + throw new IOException("Failed to simulate connection loss", e); + } + lastResetBucket = currentBucket; + } + } + assertEquals("Wrong amount of data found from InputStream", numBytes, byteCount); + } + } +} diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc index e6fa3e4d4039..92f5980d4888 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc @@ -383,7 +383,7 @@ If the status is anything other than "success", an error message will explain wh Solr provides a repository abstraction to allow users to backup and restore their data to a variety of different storage systems. For example, a Solr cluster running on a local filesystem (e.g., EXT3) can store backup data on the same disk, on a remote network-mounted drive, or in some popular "cloud storage" providers, depending on the 'repository' implementation chosen. -Solr offers multiple different repository implementations out of the box (`LocalFileSystemRepository`, `GCSBackupRepository` and `S3BackupRepository`), and allows users to create plugins for their own storage systems as needed. It is also possible to create a `DelegatingBackupRepository` that delegates to another `BackupRepository` and adds or modifies some behavior on top of it. +Solr offers multiple different repository implementations out of the box (`LocalFileSystemRepository`, `GCSBackupRepository`, `S3BackupRepository`, and `BlobBackupRepository`), and allows users to create plugins for their own storage systems as needed. It is also possible to create a `DelegatingBackupRepository` that delegates to another `BackupRepository` and adds or modifies some behavior on top of it. Users can define any number of repositories in their `solr.xml` file. The backup and restore APIs described above allow users to select which of these definitions they want to use at runtime via the `repository` parameter. @@ -794,3 +794,253 @@ https://docs.aws.amazon.com/sdkref/latest/guide/settings-global.html[These optio * Retries ** RetryMode (`LEGACY`, `STANDARD`, `ADAPTIVE`) ** Max Attempts + +=== BlobBackupRepository + +Stores and retrieves backup files in a Microsoft Azure Blob Storage container. + +This is provided via the `blob-repository` xref:configuration-guide:solr-modules.adoc[Solr Module] that needs to be enabled before use. + +BlobBackupRepository supports four authentication methods, each suitable for different deployment scenarios: + +==== Authentication Methods + +*Connection String* (recommended for development/testing):: ++ +The simplest authentication method using a complete Azure Storage connection string. +Ideal for local development with Azurite emulator or quick testing. ++ +[source,xml] +---- + + + solr-backup + DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net + + +---- + +*Account Name + Access Key* (recommended for simple production):: ++ +Separates the account name from the access key, providing cleaner configuration and easier credential rotation. ++ +[source,xml] +---- + + + solr-backup + myaccount + mykey + + +---- + +*Shared Access Signature (SAS) Token* (recommended for production with time-limited access):: ++ +Provides time-limited, permission-scoped access without exposing account keys. +SAS tokens must include service, container, and object permissions (`srt=sco`) with read, write, delete, list, add, and create permissions (`sp=rwdlac`). ++ +The container must be pre-created before using a SAS token. ++ +[source,xml] +---- + + + solr-backup + myaccount + sv=2024-11-04&ss=b&srt=sco&sp=rwdlacytfx&se=2025-12-31T23:59:59Z&st=2025-01-01T00:00:00Z&spr=https&sig=... + + +---- ++ +NOTE: SAS tokens in XML must have `&` characters escaped as `&`. + +*Azure Identity* (recommended for production on Azure infrastructure):: ++ +Uses Azure Active Directory (Azure Entra ID) authentication, supporting Managed Identities, Service Principals, and Azure CLI credentials. +This is the most secure option for production deployments running on Azure infrastructure. ++ +For *Managed Identity* (for VMs, AKS, App Service): ++ +[source,xml] +---- + + + solr-backup + https://myaccount.blob.core.windows.net + + +---- ++ +For *Service Principal*: ++ +[source,xml] +---- + + + solr-backup + https://myaccount.blob.core.windows.net + your-tenant-id + your-client-id + your-client-secret + + +---- ++ +For *Azure CLI* (development only): ++ +[source,xml] +---- + + + solr-backup + https://myaccount.blob.core.windows.net + + +---- ++ +NOTE: When using Azure Identity, the identity must have the "Storage Blob Data Contributor" role assigned to the storage account. + +==== Configuration Options + +BlobBackupRepository accepts the following configuration options: + +`blob.container.name`:: ++ +[%autowidth,frame=none] +|=== +|Required |Default: none +|=== ++ +The name of the Azure Blob Storage container to use for backups. +The container must exist before performing backup operations. + +`blob.connection.string`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Complete Azure Storage connection string including account name, key, and endpoints. +Required for Connection String authentication. +Mutually exclusive with other authentication methods. + +`blob.account.name`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Azure Storage account name. +Required for Account Name + Key and SAS Token authentication methods. + +`blob.account.key`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Azure Storage account access key. +Required for Account Name + Key authentication. +Mutually exclusive with SAS token and Azure Identity. + +`blob.sas.token`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Shared Access Signature token for time-limited, permission-scoped access. +Must include `srt=sco` (service, container, object) and `sp=rwdlac` permissions. +The `&` characters must be XML-escaped as `&` in `solr.xml`. +Mutually exclusive with account key and Azure Identity. + +`blob.endpoint`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Azure Blob Storage endpoint URL in the format `https://.blob.core.windows.net`. +Required for Azure Identity authentication. +Can be used with other methods to override default endpoint. + +`azure.tenant.id`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Azure Active Directory tenant ID. +Required for Service Principal authentication. + +`azure.client.id`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Azure Active Directory application (client) ID. +Required for Service Principal authentication. + +`azure.client.secret`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Azure Active Directory application (client) secret. +Required for Service Principal authentication. + +`location`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +A default path prefix within the container for backup storage. +Used as a fallback when users don't provide a `location` parameter in their Backup or Restore API commands. +Can be `/` to use the root of the container. + +==== Local Development with Azurite + +For local development and testing, BlobBackupRepository works with the Azurite emulator, which provides a local Azure Storage-compatible environment. + +Install and start Azurite: +[source,bash] +---- +npm install -g azurite +azurite --blobPort 10000 +---- + +Configure `solr.xml` with Azurite connection string: +[source,xml] +---- + + + solr-backup + DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1; + + +---- + +==== Production Deployment Best Practices + +* *Use Azure Identity (Managed Identity)* for production deployments on Azure VMs, AKS, or App Service +* *Use SAS tokens* for production deployments outside Azure or when time-limited access is required +* *Avoid Connection String and Account Keys* in production as they provide unlimited access +* *Enable soft delete* on your Azure Storage account for data protection +* *Use lifecycle management* to automatically archive or delete old backups +* *Monitor backup operations* through Azure Storage metrics and logs +* *Test restore operations* regularly to ensure backup integrity + +For more detailed information on Azure authentication setup, SAS token generation, and troubleshooting, refer to the module documentation in `solr/modules/blob-repository/README.md`. From 47f456afd933d9529f015ae1cc58c36fbf747c08 Mon Sep 17 00:00:00 2001 From: Prateek Singhal Date: Fri, 21 Nov 2025 14:49:05 -0800 Subject: [PATCH 2/5] SOLR-17949: Rename module to azure-blob-repository and refactor classes - Renamed module from blob-repository to azure-blob-repository - Renamed all classes from Blob* to AzureBlob* for clarity - Updated package from org.apache.solr.blob to org.apache.solr.azureblob - Added Azure SDK dependencies (azure-storage-blob, azure-identity) - Updated Solr Reference Guide with Azure Blob Storage documentation - Added .gitignore entries for Azurite test infrastructure All authentication methods tested successfully with real Azure Blob Storage: - Connection String authentication - Account Name + Key authentication - SAS Token authentication - Service Principal (Azure Identity) authentication Testing completed with 100% success rate on backup/restore operations. --- .gitignore | 5 + gradle/libs.versions.toml | 8 +- settings.gradle | 2 +- solr/licenses/azure-NOTICE.txt | 25 +---- solr/licenses/msal4j-NOTICE.txt | 25 +---- solr/licenses/reactor-NOTICE.txt | 28 +---- .../README.md | 104 ++++++++++-------- .../build.gradle | 0 .../azureblob/AzureBlobBackupRepository.java} | 38 +++---- .../AzureBlobBackupRepositoryConfig.java} | 30 ++--- .../solr/azureblob/AzureBlobException.java} | 10 +- .../solr/azureblob/AzureBlobIndexInput.java} | 18 +-- .../AzureBlobNotFoundException.java} | 6 +- .../azureblob/AzureBlobOutputStream.java} | 13 ++- .../azureblob/AzureBlobStorageClient.java} | 59 +++++----- .../apache/solr/azureblob}/package-info.java | 15 ++- .../src/test-files/conf/schema.xml | 0 .../src/test-files/conf/solrconfig.xml | 0 .../src/test-files/log4j2.xml | 0 .../AbstractAzureBlobClientTest.java} | 16 +-- .../AzureBlobBackupRepositoryTest.java} | 14 +-- .../AzureBlobIncrementalBackupTest.java} | 6 +- .../azureblob/AzureBlobIndexInputTest.java} | 28 ++--- .../azureblob/AzureBlobInstallShardTest.java} | 4 +- .../azureblob/AzureBlobOutputStreamTest.java} | 4 +- .../solr/azureblob/AzureBlobPathsTest.java} | 16 +-- .../azureblob/AzureBlobReadWriteTest.java} | 6 +- .../pages/backup-restore.adoc | 78 ++++++------- 28 files changed, 270 insertions(+), 288 deletions(-) rename solr/modules/{blob-repository => azure-blob-repository}/README.md (80%) rename solr/modules/{blob-repository => azure-blob-repository}/build.gradle (100%) rename solr/modules/{blob-repository/src/java/org/apache/solr/blob/BlobBackupRepository.java => azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobBackupRepository.java} (92%) rename solr/modules/{blob-repository/src/java/org/apache/solr/blob/BlobBackupRepositoryConfig.java => azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobBackupRepositoryConfig.java} (71%) rename solr/modules/{blob-repository/src/java/org/apache/solr/blob/BlobException.java => azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobException.java} (77%) rename solr/modules/{blob-repository/src/java/org/apache/solr/blob/BlobIndexInput.java => azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobIndexInput.java} (92%) rename solr/modules/{blob-repository/src/java/org/apache/solr/blob/BlobNotFoundException.java => azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobNotFoundException.java} (83%) rename solr/modules/{blob-repository/src/java/org/apache/solr/blob/BlobOutputStream.java => azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobOutputStream.java} (94%) rename solr/modules/{blob-repository/src/java/org/apache/solr/blob/BlobStorageClient.java => azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobStorageClient.java} (90%) rename solr/modules/{blob-repository/src/java/org/apache/solr/blob => azure-blob-repository/src/java/org/apache/solr/azureblob}/package-info.java (70%) rename solr/modules/{blob-repository => azure-blob-repository}/src/test-files/conf/schema.xml (100%) rename solr/modules/{blob-repository => azure-blob-repository}/src/test-files/conf/solrconfig.xml (100%) rename solr/modules/{blob-repository => azure-blob-repository}/src/test-files/log4j2.xml (100%) rename solr/modules/{blob-repository/src/test/org/apache/solr/blob/AbstractBlobClientTest.java => azure-blob-repository/src/test/org/apache/solr/azureblob/AbstractAzureBlobClientTest.java} (92%) rename solr/modules/{blob-repository/src/test/org/apache/solr/blob/BlobBackupRepositoryTest.java => azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobBackupRepositoryTest.java} (96%) rename solr/modules/{blob-repository/src/test/org/apache/solr/blob/BlobIncrementalBackupTest.java => azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobIncrementalBackupTest.java} (98%) rename solr/modules/{blob-repository/src/test/org/apache/solr/blob/BlobIndexInputTest.java => azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobIndexInputTest.java} (86%) rename solr/modules/{blob-repository/src/test/org/apache/solr/blob/BlobInstallShardTest.java => azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobInstallShardTest.java} (98%) rename solr/modules/{blob-repository/src/test/org/apache/solr/blob/BlobOutputStreamTest.java => azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobOutputStreamTest.java} (98%) rename solr/modules/{blob-repository/src/test/org/apache/solr/blob/BlobPathsTest.java => azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobPathsTest.java} (96%) rename solr/modules/{blob-repository/src/test/org/apache/solr/blob/BlobReadWriteTest.java => azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobReadWriteTest.java} (98%) diff --git a/.gitignore b/.gitignore index 05199687470f..2c360163ad0e 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,8 @@ gradle/wrapper/gradle-wrapper.jar # WANT TO ADD MORE? You can tell Git without adding to this file: # See https://git-scm.com/docs/gitignore # In particular, if you have tools you use, add to $GIT_DIR/info/exclude or use core.excludesFile + +# Azure Blob Storage testing artifacts (local testing only) +AzuriteConfig +__azurite_db_*.json +__blobstorage__/ diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index c63808a26071..f7627548e280 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -50,10 +50,10 @@ aqute-bnd = "6.4.1" asciidoctor-mathjax = "0.0.9" # @keep Asciidoctor tabs version used in ref-guide asciidoctor-tabs = "1.0.0-beta.6" -azure-storage = "12.25.0" -azure-identity = "1.12.0" azure-core = "1.52.0" azure-core-http-netty = "1.15.4" +azure-identity = "1.12.0" +azure-storage = "12.25.0" # @keep bats-assert (node) version used in packaging bats-assert = "2.0.0" # @keep bats-core (node) version used in packaging @@ -308,10 +308,10 @@ apache-zookeeper-zookeeper = { module = "org.apache.zookeeper:zookeeper", versio # @keep transitive dependency for version alignment apiguardian-api = { module = "org.apiguardian:apiguardian-api", version.ref = "apiguardian" } aqute-bnd-annotation = { module = "biz.aQute.bnd:biz.aQute.bnd.annotation", version.ref = "aqute-bnd" } -azure-storage-blob = { module = "com.azure:azure-storage-blob", version.ref = "azure-storage" } -azure-identity = { module = "com.azure:azure-identity", version.ref = "azure-identity" } azure-core = { module = "com.azure:azure-core", version.ref = "azure-core" } azure-core-http-netty = { module = "com.azure:azure-core-http-netty", version.ref = "azure-core-http-netty" } +azure-identity = { module = "com.azure:azure-identity", version.ref = "azure-identity" } +azure-storage-blob = { module = "com.azure:azure-storage-blob", version.ref = "azure-storage" } bc-jose4j = { module = "org.bitbucket.b_c:jose4j", version.ref = "bc-jose4j" } benmanes-caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version.ref = "benmanes-caffeine" } bouncycastle-bcpkix = { module = "org.bouncycastle:bcpkix-jdk18on", version.ref = "bouncycastle" } diff --git a/settings.gradle b/settings.gradle index eff852fb9c65..16574e6e4956 100644 --- a/settings.gradle +++ b/settings.gradle @@ -44,7 +44,7 @@ include "solr:core" include "solr:cross-dc-manager" include "solr:server" include "solr:modules:analysis-extras" -include "solr:modules:blob-repository" +include "solr:modules:azure-blob-repository" include "solr:modules:clustering" include "solr:modules:cross-dc" include "solr:modules:cuvs" diff --git a/solr/licenses/azure-NOTICE.txt b/solr/licenses/azure-NOTICE.txt index 7b5a06890325..2831e601401c 100644 --- a/solr/licenses/azure-NOTICE.txt +++ b/solr/licenses/azure-NOTICE.txt @@ -1,25 +1,12 @@ -AWS SDK for Java 2.0 -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +Azure SDK for Java +Copyright (c) Microsoft Corporation. All rights reserved. This product includes software developed by -Amazon Technologies, Inc (http://www.amazon.com/). +Microsoft Corporation (https://github.com/Azure/azure-sdk-for-java). + +Licensed under the MIT License. ********************** THIRD PARTY COMPONENTS ********************** -This software includes third party software subject to the following copyrights: -- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. -- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. -- Apache Commons Lang - https://github.com/apache/commons-lang -- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams -- Jackson-core - https://github.com/FasterXML/jackson-core -- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary - -The licenses for these third party components are included in LICENSE.txt - -- For Apache Commons Lang see also this required NOTICE: - Apache Commons Lang - Copyright 2001-2020 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (https://www.apache.org/). +This software may include third party software subject to the following copyrights: diff --git a/solr/licenses/msal4j-NOTICE.txt b/solr/licenses/msal4j-NOTICE.txt index 7b5a06890325..cf861d18eea7 100644 --- a/solr/licenses/msal4j-NOTICE.txt +++ b/solr/licenses/msal4j-NOTICE.txt @@ -1,25 +1,12 @@ -AWS SDK for Java 2.0 -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +Microsoft Authentication Library for Java (MSAL4J) +Copyright (c) Microsoft Corporation. All rights reserved. This product includes software developed by -Amazon Technologies, Inc (http://www.amazon.com/). +Microsoft Corporation (https://github.com/AzureAD/microsoft-authentication-library-for-java). + +Licensed under the MIT License. ********************** THIRD PARTY COMPONENTS ********************** -This software includes third party software subject to the following copyrights: -- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. -- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. -- Apache Commons Lang - https://github.com/apache/commons-lang -- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams -- Jackson-core - https://github.com/FasterXML/jackson-core -- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary - -The licenses for these third party components are included in LICENSE.txt - -- For Apache Commons Lang see also this required NOTICE: - Apache Commons Lang - Copyright 2001-2020 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (https://www.apache.org/). +This software may include third party software subject to the following copyrights: diff --git a/solr/licenses/reactor-NOTICE.txt b/solr/licenses/reactor-NOTICE.txt index 7b5a06890325..990ac4433824 100644 --- a/solr/licenses/reactor-NOTICE.txt +++ b/solr/licenses/reactor-NOTICE.txt @@ -1,25 +1,7 @@ -AWS SDK for Java 2.0 -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +Project Reactor +Copyright (c) 2011-2024 VMware Inc. or its affiliates, All Rights Reserved. -This product includes software developed by -Amazon Technologies, Inc (http://www.amazon.com/). +This product includes software developed at +VMware Inc. (https://github.com/reactor) -********************** -THIRD PARTY COMPONENTS -********************** -This software includes third party software subject to the following copyrights: -- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. -- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. -- Apache Commons Lang - https://github.com/apache/commons-lang -- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams -- Jackson-core - https://github.com/FasterXML/jackson-core -- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary - -The licenses for these third party components are included in LICENSE.txt - -- For Apache Commons Lang see also this required NOTICE: - Apache Commons Lang - Copyright 2001-2020 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (https://www.apache.org/). +Licensed under the Apache License 2.0 diff --git a/solr/modules/blob-repository/README.md b/solr/modules/azure-blob-repository/README.md similarity index 80% rename from solr/modules/blob-repository/README.md rename to solr/modules/azure-blob-repository/README.md index 3deab497a674..5c7b573d14cf 100644 --- a/solr/modules/blob-repository/README.md +++ b/solr/modules/azure-blob-repository/README.md @@ -32,7 +32,7 @@ This Azure Blob Storage repository is a backup repository implementation designe **Prerequisites:** - Azure Storage Account with a blob container - Container must already exist (e.g., `solr-backup`) -- Solr blob-repository module enabled +- Solr azure-blob-repository module enabled - Network access to Azure Blob Storage (HTTPS port 443) ## Prerequisites @@ -47,9 +47,9 @@ Before configuring authentication, ensure you have: --name solr-backup \ --account-name YOUR_ACCOUNT_NAME ``` -3. **Solr Module** - Enable blob-repository module: +3. **Solr Module** - Enable azure-blob-repository module: ```bash - export SOLR_MODULES=blob-repository + export SOLR_MODULES=azure-blob-repository ./bin/solr start ``` 4. **Network Access** - Solr can reach Azure Blob Storage (HTTPS port 443) @@ -68,12 +68,16 @@ The Azure Blob Storage backup repository supports four authentication methods. C The simplest authentication method using a full connection string. #### Configuration in solr.xml: + ```xml + - - YOUR_CONTAINER_NAME - DefaultEndpointsProtocol=https;AccountName=YOUR_ACCOUNT_NAME;AccountKey=YOUR_ACCOUNT_KEY;EndpointSuffix=core.windows.net - + + YOUR_CONTAINER_NAME + + DefaultEndpointsProtocol=https;AccountName=YOUR_ACCOUNT_NAME;AccountKey=YOUR_ACCOUNT_KEY;EndpointSuffix=core.windows.net + + ``` @@ -84,14 +88,16 @@ The simplest authentication method using a full connection string. Separates the account credentials from the endpoint configuration. #### Configuration in solr.xml: + ```xml + - - YOUR_CONTAINER_NAME - https://YOUR_ACCOUNT_NAME.blob.core.windows.net - YOUR_ACCOUNT_NAME - YOUR_ACCOUNT_KEY - + + YOUR_CONTAINER_NAME + https://YOUR_ACCOUNT_NAME.blob.core.windows.net + YOUR_ACCOUNT_NAME + YOUR_ACCOUNT_KEY + ``` @@ -133,13 +139,15 @@ az storage account generate-sas \ ``` #### Configuration in solr.xml: + ```xml + - - YOUR_CONTAINER_NAME - https://YOUR_ACCOUNT_NAME.blob.core.windows.net - sv=2024-11-04&ss=b&srt=sco&sp=rwdlac&se=2026-12-31T23:59:59Z&st=2025-01-01T00:00:00Z&spr=https&sig=YOUR_SIGNATURE - + + YOUR_CONTAINER_NAME + https://YOUR_ACCOUNT_NAME.blob.core.windows.net + sv=2024-11-04&ss=b&srt=sco&sp=rwdlac&se=2026-12-31T23:59:59Z&st=2025-01-01T00:00:00Z&spr=https&sig=YOUR_SIGNATURE + ``` @@ -184,13 +192,15 @@ az role assignment create \ ``` **Configuration in solr.xml:** + ```xml + - - YOUR_CONTAINER_NAME - https://YOUR_ACCOUNT_NAME.blob.core.windows.net - - + + YOUR_CONTAINER_NAME + https://YOUR_ACCOUNT_NAME.blob.core.windows.net + + ``` @@ -216,15 +226,17 @@ az ad sp create-for-rbac \ ``` **Configuration in solr.xml:** + ```xml + - - YOUR_CONTAINER_NAME - https://YOUR_ACCOUNT_NAME.blob.core.windows.net - YOUR_TENANT_ID - YOUR_CLIENT_ID - YOUR_CLIENT_SECRET - + + YOUR_CONTAINER_NAME + https://YOUR_ACCOUNT_NAME.blob.core.windows.net + YOUR_TENANT_ID + YOUR_CLIENT_ID + YOUR_CLIENT_SECRET + ``` @@ -238,13 +250,15 @@ export AZURE_CLIENT_SECRET="your-client-secret" ``` Then solr.xml only needs: + ```xml + - - YOUR_CONTAINER_NAME - https://YOUR_ACCOUNT_NAME.blob.core.windows.net - - + + YOUR_CONTAINER_NAME + https://YOUR_ACCOUNT_NAME.blob.core.windows.net + + ``` @@ -275,13 +289,15 @@ az role assignment create \ ``` **Configuration in solr.xml:** + ```xml + - - YOUR_CONTAINER_NAME - https://YOUR_ACCOUNT_NAME.blob.core.windows.net - - + + YOUR_CONTAINER_NAME + https://YOUR_ACCOUNT_NAME.blob.core.windows.net + + ``` @@ -384,7 +400,7 @@ Once you've configured authentication in `solr.xml`, you can use standard Solr b ```bash # Create a backup of a collection -curl "http://localhost:8983/solr/admin/collections?action=BACKUP&name=my-backup&collection=my-collection&repository=blob&location=/" +curl "http://localhost:8983/solr/admin/collections?action=BACKUP&name=my-backup&collection=my-collection&repository=azure_blob&location=/" # Example response: # { @@ -408,7 +424,7 @@ curl "http://localhost:8983/solr/admin/collections?action=BACKUP&name=my-backup& ```bash # Restore a backup to a new or existing collection -curl "http://localhost:8983/solr/admin/collections?action=RESTORE&name=my-backup&collection=my-collection-restored&repository=blob&location=/" +curl "http://localhost:8983/solr/admin/collections?action=RESTORE&name=my-backup&collection=my-collection-restored&repository=azure_blob&location=/" # Example response: # { @@ -427,7 +443,7 @@ curl "http://localhost:8983/solr/admin/collections?action=RESTORE&name=my-backup ```bash # List all backups at a location -curl "http://localhost:8983/solr/admin/collections?action=LISTBACKUP&name=my-backup&repository=blob&location=/" +curl "http://localhost:8983/solr/admin/collections?action=LISTBACKUP&name=my-backup&repository=azure_blob&location=/" # Example response: # { @@ -443,7 +459,7 @@ curl "http://localhost:8983/solr/admin/collections?action=LISTBACKUP&name=my-bac ```bash # Delete a specific backup -curl "http://localhost:8983/solr/admin/collections?action=DELETEBACKUP&name=my-backup&backupId=1&repository=blob&location=/" +curl "http://localhost:8983/solr/admin/collections?action=DELETEBACKUP&name=my-backup&backupId=1&repository=azure_blob&location=/" ``` **Note:** The `location` parameter should be `/` (root of container) or a subdirectory path like `/backups/`. The path must not have a trailing slash except for root. diff --git a/solr/modules/blob-repository/build.gradle b/solr/modules/azure-blob-repository/build.gradle similarity index 100% rename from solr/modules/blob-repository/build.gradle rename to solr/modules/azure-blob-repository/build.gradle diff --git a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobBackupRepository.java b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobBackupRepository.java similarity index 92% rename from solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobBackupRepository.java rename to solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobBackupRepository.java index 54bef83bfefb..934798ee7d0f 100644 --- a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobBackupRepository.java +++ b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobBackupRepository.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.blob; +package org.apache.solr.azureblob; import java.io.IOException; import java.io.InputStream; @@ -44,19 +44,19 @@ * A concrete implementation of {@link BackupRepository} interface supporting backup/restore of Solr * indexes to Azure Blob Storage. */ -public class BlobBackupRepository extends AbstractBackupRepository { +public class AzureBlobBackupRepository extends AbstractBackupRepository { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); static final String BLOB_SCHEME = "blob"; private static final int CHUNK_SIZE = 16 * 1024 * 1024; - private BlobStorageClient client; + private AzureBlobStorageClient client; @Override public void init(NamedList args) { super.init(args); - BlobBackupRepositoryConfig backupConfig = new BlobBackupRepositoryConfig(this.config); + AzureBlobBackupRepositoryConfig backupConfig = new AzureBlobBackupRepositoryConfig(this.config); // If a client was already created, close it to avoid any resource leak if (client != null) { @@ -67,7 +67,7 @@ public void init(NamedList args) { } // Method to inject a mock client for testing - public void setClient(BlobStorageClient client) { + public void setClient(AzureBlobStorageClient client) { this.client = client; } @@ -147,7 +147,7 @@ public void createDirectory(URI path) throws IOException { try { client.createDirectory(blobPath); - } catch (BlobException e) { + } catch (AzureBlobException e) { throw new IOException("Failed to create directory " + blobPath, e); } } @@ -164,7 +164,7 @@ public void deleteDirectory(URI path) throws IOException { try { client.deleteDirectory(blobPath); - } catch (BlobException e) { + } catch (AzureBlobException e) { throw new IOException("Failed to delete directory " + blobPath, e); } } @@ -181,7 +181,7 @@ public void delete(URI path, Collection files) throws IOException { int lastSlash = basePath.lastIndexOf('/'); basePath = lastSlash >= 0 ? basePath.substring(0, lastSlash) : ""; } - } catch (BlobException e) { + } catch (AzureBlobException e) { throw new IOException("Failed to check path type for " + basePath, e); } @@ -197,7 +197,7 @@ public void delete(URI path, Collection files) throws IOException { try { client.delete(fullPaths); - } catch (BlobException e) { + } catch (AzureBlobException e) { throw new IOException("Failed to delete files " + fullPaths, e); } } @@ -214,7 +214,7 @@ public boolean exists(URI path) throws IOException { try { return client.pathExists(blobPath); - } catch (BlobException e) { + } catch (AzureBlobException e) { throw new IOException("Failed to check existence of " + blobPath, e); } } @@ -235,7 +235,7 @@ public PathType getPathType(URI path) throws IOException { } else { return BackupRepository.PathType.FILE; } - } catch (BlobException e) { + } catch (AzureBlobException e) { throw new IOException("Failed to get path type for " + blobPath, e); } } @@ -252,7 +252,7 @@ public String[] listAll(URI path) throws IOException { try { return client.listDir(blobPath); - } catch (BlobException e) { + } catch (AzureBlobException e) { throw new IOException("Failed to list directory " + blobPath, e); } } @@ -270,8 +270,8 @@ public IndexInput openInput(URI dirPath, String fileName, IOContext ctx) throws } try { - return new BlobIndexInput(blobPath, client, client.length(blobPath)); - } catch (BlobException e) { + return new AzureBlobIndexInput(blobPath, client, client.length(blobPath)); + } catch (AzureBlobException e) { throw new IOException("Failed to open input stream for " + blobPath, e); } } @@ -288,7 +288,7 @@ public OutputStream createOutput(URI path) throws IOException { try { return client.pushStream(blobPath); - } catch (BlobException e) { + } catch (AzureBlobException e) { throw new IOException("Failed to create output stream for " + blobPath, e); } } @@ -316,7 +316,7 @@ public void copyIndexFileFrom( if (!parentDir.isEmpty()) { client.createDirectory(parentDir); } - } catch (BlobException e) { + } catch (AzureBlobException e) { // ignore failures here; write will surface real issues } @@ -331,7 +331,7 @@ public void copyIndexFileFrom( output.write(buffer, 0, toRead); remaining -= toRead; } - } catch (BlobException e) { + } catch (AzureBlobException e) { throw new IOException("Failed to copy file from " + sourceFileName + " to " + blobPath, e); } } @@ -380,14 +380,14 @@ public void copyIndexFileTo( while ((len = inputStream.read(buffer)) != -1) { indexOutput.writeBytes(buffer, 0, len); } - } catch (BlobException e) { + } catch (AzureBlobException e) { throw new IOException("Failed to copy file from " + blobPath + " to " + destFileName, e); } long timeElapsed = Duration.between(start, Instant.now()).toMillis(); if (log.isInfoEnabled()) { - log.info("Download from S3 '{}' finished in {}ms", blobPath, timeElapsed); + log.info("Download from Azure Blob Storage '{}' finished in {}ms", blobPath, timeElapsed); } } diff --git a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobBackupRepositoryConfig.java b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobBackupRepositoryConfig.java similarity index 71% rename from solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobBackupRepositoryConfig.java rename to solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobBackupRepositoryConfig.java index 59558c3bf7d8..ef960dfe9f22 100644 --- a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobBackupRepositoryConfig.java +++ b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobBackupRepositoryConfig.java @@ -14,23 +14,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.blob; +package org.apache.solr.azureblob; import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.NamedList; /** Class representing the {@code backup} Blob Storage config bundle specified in solr.xml. */ -public class BlobBackupRepositoryConfig { +public class AzureBlobBackupRepositoryConfig { - public static final String CONTAINER_NAME = "blob.container.name"; - public static final String CONNECTION_STRING = "blob.connection.string"; - public static final String ENDPOINT = "blob.endpoint"; - public static final String ACCOUNT_NAME = "blob.account.name"; - public static final String ACCOUNT_KEY = "blob.account.key"; - public static final String SAS_TOKEN = "blob.sas.token"; - public static final String TENANT_ID = "blob.tenant.id"; - public static final String CLIENT_ID = "blob.client.id"; - public static final String CLIENT_SECRET = "blob.client.secret"; + public static final String CONTAINER_NAME = "azure.blob.container.name"; + public static final String CONNECTION_STRING = "azure.blob.connection.string"; + public static final String ENDPOINT = "azure.blob.endpoint"; + public static final String ACCOUNT_NAME = "azure.blob.account.name"; + public static final String ACCOUNT_KEY = "azure.blob.account.key"; + public static final String SAS_TOKEN = "azure.blob.sas.token"; + public static final String TENANT_ID = "azure.blob.tenant.id"; + public static final String CLIENT_ID = "azure.blob.client.id"; + public static final String CLIENT_SECRET = "azure.blob.client.secret"; private final String containerName; private final String connectionString; @@ -42,7 +42,7 @@ public class BlobBackupRepositoryConfig { private final String clientId; private final String clientSecret; - public BlobBackupRepositoryConfig(NamedList config) { + public AzureBlobBackupRepositoryConfig(NamedList config) { containerName = getStringConfig(config, CONTAINER_NAME); connectionString = getStringConfig(config, CONNECTION_STRING); endpoint = getStringConfig(config, ENDPOINT); @@ -54,9 +54,9 @@ public BlobBackupRepositoryConfig(NamedList config) { clientSecret = getStringConfig(config, CLIENT_SECRET); } - /** Construct a {@link BlobStorageClient} from the provided config. */ - public BlobStorageClient buildClient() { - return new BlobStorageClient( + /** Construct a {@link AzureBlobStorageClient} from the provided config. */ + public AzureBlobStorageClient buildClient() { + return new AzureBlobStorageClient( containerName, connectionString, endpoint, diff --git a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobException.java b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobException.java similarity index 77% rename from solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobException.java rename to solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobException.java index 62890aa60efb..f32700351fab 100644 --- a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobException.java +++ b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobException.java @@ -14,18 +14,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.blob; +package org.apache.solr.azureblob; /** * Generic exception for Blob Storage related failures. Could originate from the {@link - * BlobBackupRepository} or from its underlying {@link BlobStorageClient}. + * AzureBlobBackupRepository} or from its underlying {@link AzureBlobStorageClient}. */ -public class BlobException extends Exception { - public BlobException(String message) { +public class AzureBlobException extends Exception { + public AzureBlobException(String message) { super(message); } - public BlobException(String message, Throwable cause) { + public AzureBlobException(String message, Throwable cause) { super(message, cause); } } diff --git a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobIndexInput.java b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobIndexInput.java similarity index 92% rename from solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobIndexInput.java rename to solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobIndexInput.java index 1938f24f2e3f..fc935b2a4c52 100644 --- a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobIndexInput.java +++ b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobIndexInput.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.blob; +package org.apache.solr.azureblob; import java.io.EOFException; import java.io.IOException; @@ -23,13 +23,13 @@ import java.util.Map; import org.apache.lucene.store.IndexInput; -class BlobIndexInput extends IndexInput { +class AzureBlobIndexInput extends IndexInput { private static final int DEFAULT_PAGE_SIZE = 512 * 1024; // 512 KB private static final int MAX_CACHED_PAGES = 128; // ~64 MB at 512 KB pages private final String path; - private final BlobStorageClient client; + private final AzureBlobStorageClient client; private final long length; private final int pageSize; private final LruPageCache cache; @@ -37,12 +37,12 @@ class BlobIndexInput extends IndexInput { private long position = 0L; private boolean closed = false; - BlobIndexInput(String path, BlobStorageClient client, long length) { + AzureBlobIndexInput(String path, AzureBlobStorageClient client, long length) { this(path, client, length, DEFAULT_PAGE_SIZE, MAX_CACHED_PAGES); } - BlobIndexInput( - String path, BlobStorageClient client, long length, int pageSize, int maxCachedPages) { + AzureBlobIndexInput( + String path, AzureBlobStorageClient client, long length, int pageSize, int maxCachedPages) { super(path); this.path = path; this.client = client; @@ -84,8 +84,8 @@ public IndexInput slice(String sliceDescription, long offset, long length) throw throw new IOException("Slice out of bounds: offset=" + offset + ", length=" + length); } - BlobIndexInput slice = - new BlobIndexInput( + AzureBlobIndexInput slice = + new AzureBlobIndexInput( getFullSliceDescription(sliceDescription), client, length, pageSize, MAX_CACHED_PAGES); slice.position = 0L; @@ -160,7 +160,7 @@ private byte[] getPage(long pageIdx) throws IOException { throw new EOFException( "End of stream reached: expected " + bytesToRead + " bytes, got " + readTotal); } - } catch (BlobException e) { + } catch (AzureBlobException e) { throw new IOException("Failed to fetch range page", e); } diff --git a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobNotFoundException.java b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobNotFoundException.java similarity index 83% rename from solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobNotFoundException.java rename to solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobNotFoundException.java index 88e0c41e781d..a6f5253c0e3f 100644 --- a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobNotFoundException.java +++ b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobNotFoundException.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.blob; +package org.apache.solr.azureblob; /** Exception thrown when a blob is not found in Azure Blob Storage. */ -public class BlobNotFoundException extends BlobException { - public BlobNotFoundException(String message, Throwable cause) { +public class AzureBlobNotFoundException extends AzureBlobException { + public AzureBlobNotFoundException(String message, Throwable cause) { super(message, cause); } } diff --git a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobOutputStream.java b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobOutputStream.java similarity index 94% rename from solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobOutputStream.java rename to solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobOutputStream.java index 41a9b3fe0a10..61388c983889 100644 --- a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobOutputStream.java +++ b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobOutputStream.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.blob; +package org.apache.solr.azureblob; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.models.BlobStorageException; @@ -36,7 +36,7 @@ * OutputStream implementation for Azure Blob Storage using block blobs. Supports chunked uploads * for large files. */ -public class BlobOutputStream extends OutputStream { +public class AzureBlobOutputStream extends OutputStream { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); // 4 MB per block (Azure limit is 100 MB, but 4 MB is more efficient for most use cases) @@ -49,7 +49,7 @@ public class BlobOutputStream extends OutputStream { private BlockUpload blockUpload; private boolean committed; - public BlobOutputStream(BlobClient blobClient, String blobPath) { + public AzureBlobOutputStream(BlobClient blobClient, String blobPath) { this.blobClient = blobClient; this.blobPath = blobPath; this.closed = false; @@ -132,7 +132,8 @@ private void uploadBlock() throws IOException { log.debug("Block upload aborted for blobPath '{}'.", blobPath); } } - throw new IOException("Failed to upload block", BlobStorageClient.handleBlobException(e)); + throw new IOException( + "Failed to upload block", AzureBlobStorageClient.handleBlobException(e)); } // reset the buffer for eventual next write operation @@ -183,7 +184,7 @@ public void close() throws IOException { blobClient.upload(new ByteArrayInputStream(new byte[0]), 0, true); } catch (BlobStorageException e) { throw new IOException( - "Failed to create empty blob", BlobStorageClient.handleBlobException(e)); + "Failed to create empty blob", AzureBlobStorageClient.handleBlobException(e)); } } } else { @@ -202,7 +203,7 @@ private BlockUpload newBlockUpload() throws IOException { return new BlockUpload(); } catch (BlobStorageException e) { throw new IOException( - "Failed to create block upload", BlobStorageClient.handleBlobException(e)); + "Failed to create block upload", AzureBlobStorageClient.handleBlobException(e)); } } diff --git a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobStorageClient.java b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobStorageClient.java similarity index 90% rename from solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobStorageClient.java rename to solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobStorageClient.java index 5ad196caae13..62606d288adb 100644 --- a/solr/modules/blob-repository/src/java/org/apache/solr/blob/BlobStorageClient.java +++ b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobStorageClient.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.blob; +package org.apache.solr.azureblob; import com.azure.core.credential.TokenCredential; import com.azure.identity.DefaultAzureCredentialBuilder; @@ -45,7 +45,7 @@ * Creates a {@link BlobServiceClient} for communicating with Azure Blob Storage. Utilizes the * default Azure credential provider chain. */ -public class BlobStorageClient { +public class AzureBlobStorageClient { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -53,7 +53,7 @@ public class BlobStorageClient { private final BlobContainerClient containerClient; - BlobStorageClient( + AzureBlobStorageClient( String containerName, String connectionString, String endpoint, @@ -77,7 +77,7 @@ public class BlobStorageClient { } @VisibleForTesting - BlobStorageClient(BlobServiceClient blobServiceClient, String containerName) { + AzureBlobStorageClient(BlobServiceClient blobServiceClient, String containerName) { this.containerClient = blobServiceClient.getBlobContainerClient(containerName); try { containerClient.create(); @@ -123,7 +123,7 @@ private static BlobServiceClient createInternalClient( } /** Create a directory in Blob Storage, if it does not already exist. */ - void createDirectory(String path) throws BlobException { + void createDirectory(String path) throws AzureBlobException { String sanitizedDirPath = sanitizedDirPath(path); // Only create the directory if it does not already exist @@ -148,7 +148,7 @@ void createDirectory(String path) throws BlobException { } /** Delete files from Blob Storage. Missing files are ignored (idempotent delete). */ - void delete(Collection paths) throws BlobException { + void delete(Collection paths) throws AzureBlobException { Set entries = new HashSet<>(); for (String path : paths) { entries.add(sanitizedFilePath(path)); @@ -157,7 +157,7 @@ void delete(Collection paths) throws BlobException { } /** Delete directory, all the files and subdirectories from Blob Storage. */ - void deleteDirectory(String path) throws BlobException { + void deleteDirectory(String path) throws AzureBlobException { path = sanitizedDirPath(path); // Get all the files and subdirectories @@ -170,7 +170,7 @@ void deleteDirectory(String path) throws BlobException { } /** List all the files and subdirectories directly under given path. */ - String[] listDir(String path) throws BlobException { + String[] listDir(String path) throws AzureBlobException { path = sanitizedDirPath(path); try { @@ -194,7 +194,7 @@ String[] listDir(String path) throws BlobException { } /** Check if path exists. */ - boolean pathExists(String path) throws BlobException { + boolean pathExists(String path) throws AzureBlobException { final String blobPath = sanitizedPath(path); // for root return true @@ -211,7 +211,7 @@ boolean pathExists(String path) throws BlobException { } /** Check if path is directory. */ - boolean isDirectory(String path) throws BlobException { + boolean isDirectory(String path) throws AzureBlobException { final String dirPrefix = sanitizedDirPath(path); try { @@ -242,7 +242,7 @@ boolean isDirectory(String path) throws BlobException { } /** Get length of file in bytes. */ - long length(String path) throws BlobException { + long length(String path) throws AzureBlobException { String blobPath = sanitizedFilePath(path); try { BlobClient blobClient = containerClient.getBlobClient(blobPath); @@ -253,7 +253,7 @@ long length(String path) throws BlobException { } /** Open a new {@link InputStream} to file for read. */ - InputStream pullStream(String path) throws BlobException { + InputStream pullStream(String path) throws AzureBlobException { final String blobPath = sanitizedFilePath(path); try { @@ -272,7 +272,7 @@ InputStream pullStream(String path) throws BlobException { long remaining = contentLength > 0 ? Math.max(0, contentLength - bytesRead) : Long.MAX_VALUE; return pullRangeStream(path, bytesRead, remaining); - } catch (BlobException e) { + } catch (AzureBlobException e) { // ResumableInputStream supplier cannot throw checked exceptions throw new RuntimeException(e); } @@ -283,7 +283,7 @@ InputStream pullStream(String path) throws BlobException { } /** Open a ranged {@link InputStream} to file for read from offset for length bytes. */ - InputStream pullRangeStream(String path, long offset, long length) throws BlobException { + InputStream pullRangeStream(String path, long offset, long length) throws AzureBlobException { final String blobPath = sanitizedFilePath(path); try { BlobClient blobClient = containerClient.getBlobClient(blobPath); @@ -385,7 +385,7 @@ private static boolean isAlreadyClosed(Throwable t) { } /** Open a new {@link OutputStream} to file for write. */ - OutputStream pushStream(String path) throws BlobException { + OutputStream pushStream(String path) throws AzureBlobException { path = sanitizedFilePath(path); if (!parentDirectoryExist(path)) { @@ -398,7 +398,7 @@ OutputStream pushStream(String path) throws BlobException { try { BlobClient blobClient = containerClient.getBlobClient(path); - return new BlobOutputStream(blobClient, path); + return new AzureBlobOutputStream(blobClient, path); } catch (BlobStorageException e) { throw handleBlobException(e); } @@ -421,7 +421,7 @@ void deleteContainerForTests() { } } - private Collection deleteBlobs(Collection paths) throws BlobException { + private Collection deleteBlobs(Collection paths) throws AzureBlobException { try { return deleteBlobs(paths, 1000); // Azure supports batch delete } catch (BlobStorageException e) { @@ -430,7 +430,8 @@ private Collection deleteBlobs(Collection paths) throws BlobExce } @VisibleForTesting - Collection deleteBlobs(Collection entries, int batchSize) throws BlobException { + Collection deleteBlobs(Collection entries, int batchSize) + throws AzureBlobException { Set deletedPaths = new HashSet<>(); for (String path : entries) { @@ -445,14 +446,14 @@ Collection deleteBlobs(Collection entries, int batchSize) throws // ignore missing continue; } - throw new BlobException("Could not delete blob with path: " + path, e); + throw new AzureBlobException("Could not delete blob with path: " + path, e); } } return deletedPaths; } - private Set listAll(String path) throws BlobException { + private Set listAll(String path) throws AzureBlobException { String prefix = sanitizedDirPath(path); try { @@ -468,7 +469,7 @@ private Set listAll(String path) throws BlobException { } } - private boolean parentDirectoryExist(String path) throws BlobException { + private boolean parentDirectoryExist(String path) throws AzureBlobException { String parentDirectory = getParentDirectory(path); if (parentDirectory.isEmpty() || parentDirectory.equals(BLOB_FILE_PATH_DELIMITER)) { @@ -493,7 +494,7 @@ private String getParentDirectory(String path) { } /** Ensures path adheres to some rules: -Doesn't start with a leading slash */ - String sanitizedPath(String path) throws BlobException { + String sanitizedPath(String path) throws AzureBlobException { String sanitizedPath = path.trim(); // Remove all leading slashes so that blob names never start with '/' while (sanitizedPath.startsWith(BLOB_FILE_PATH_DELIMITER)) { @@ -503,22 +504,22 @@ String sanitizedPath(String path) throws BlobException { } /** Ensures file path adheres to some rules */ - String sanitizedFilePath(String path) throws BlobException { + String sanitizedFilePath(String path) throws AzureBlobException { String sanitizedPath = sanitizedPath(path); if (sanitizedPath.endsWith(BLOB_FILE_PATH_DELIMITER)) { - throw new BlobException("Invalid Path. Path for file can't end with '/'"); + throw new AzureBlobException("Invalid Path. Path for file can't end with '/'"); } if (sanitizedPath.isEmpty()) { - throw new BlobException("Invalid Path. Path cannot be empty"); + throw new AzureBlobException("Invalid Path. Path cannot be empty"); } return sanitizedPath; } /** Ensures directory path adheres to some rules */ - String sanitizedDirPath(String path) throws BlobException { + String sanitizedDirPath(String path) throws AzureBlobException { String sanitizedPath = sanitizedPath(path); if (!sanitizedPath.endsWith(BLOB_FILE_PATH_DELIMITER)) { @@ -529,7 +530,7 @@ String sanitizedDirPath(String path) throws BlobException { } /** Handle Azure Blob Storage exceptions */ - static BlobException handleBlobException(BlobStorageException e) { + static AzureBlobException handleBlobException(BlobStorageException e) { String errMessage = String.format( Locale.ROOT, @@ -541,9 +542,9 @@ static BlobException handleBlobException(BlobStorageException e) { log.error(errMessage); if (e.getStatusCode() == 404) { - return new BlobNotFoundException(errMessage, e); + return new AzureBlobNotFoundException(errMessage, e); } else { - return new BlobException(errMessage, e); + return new AzureBlobException(errMessage, e); } } } diff --git a/solr/modules/blob-repository/src/java/org/apache/solr/blob/package-info.java b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/package-info.java similarity index 70% rename from solr/modules/blob-repository/src/java/org/apache/solr/blob/package-info.java rename to solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/package-info.java index bb93394a314a..8be0e21aca24 100644 --- a/solr/modules/blob-repository/src/java/org/apache/solr/blob/package-info.java +++ b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/package-info.java @@ -18,8 +18,8 @@ /** * Azure Blob Storage backup repository implementation for Apache Solr. * - *

This package provides a {@link org.apache.solr.blob.BlobBackupRepository} implementation that - * enables Solr to store and retrieve backup data from Azure Blob Storage. + *

This package provides a {@link org.apache.solr.azureblob.AzureBlobBackupRepository} + * implementation that enables Solr to store and retrieve backup data from Azure Blob Storage. * *

The repository supports various Azure authentication methods including: * @@ -33,12 +33,15 @@ *

Key components: * *

    - *
  • {@link org.apache.solr.blob.BlobBackupRepository} - Main repository implementation - *
  • {@link org.apache.solr.blob.BlobStorageClient} - Azure Blob Storage client wrapper - *
  • {@link org.apache.solr.blob.BlobBackupRepositoryConfig} - Configuration management + *
  • {@link org.apache.solr.azureblob.AzureBlobBackupRepository} - Main repository + * implementation + *
  • {@link org.apache.solr.azureblob.AzureBlobStorageClient} - Azure Blob Storage client + * wrapper + *
  • {@link org.apache.solr.azureblob.AzureBlobBackupRepositoryConfig} - Configuration + * management *
* * @see Azure Blob Storage * Documentation */ -package org.apache.solr.blob; +package org.apache.solr.azureblob; diff --git a/solr/modules/blob-repository/src/test-files/conf/schema.xml b/solr/modules/azure-blob-repository/src/test-files/conf/schema.xml similarity index 100% rename from solr/modules/blob-repository/src/test-files/conf/schema.xml rename to solr/modules/azure-blob-repository/src/test-files/conf/schema.xml diff --git a/solr/modules/blob-repository/src/test-files/conf/solrconfig.xml b/solr/modules/azure-blob-repository/src/test-files/conf/solrconfig.xml similarity index 100% rename from solr/modules/blob-repository/src/test-files/conf/solrconfig.xml rename to solr/modules/azure-blob-repository/src/test-files/conf/solrconfig.xml diff --git a/solr/modules/blob-repository/src/test-files/log4j2.xml b/solr/modules/azure-blob-repository/src/test-files/log4j2.xml similarity index 100% rename from solr/modules/blob-repository/src/test-files/log4j2.xml rename to solr/modules/azure-blob-repository/src/test-files/log4j2.xml diff --git a/solr/modules/blob-repository/src/test/org/apache/solr/blob/AbstractBlobClientTest.java b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AbstractAzureBlobClientTest.java similarity index 92% rename from solr/modules/blob-repository/src/test/org/apache/solr/blob/AbstractBlobClientTest.java rename to solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AbstractAzureBlobClientTest.java index b28833e4bf68..3aee901d2aa2 100644 --- a/solr/modules/blob-repository/src/test/org/apache/solr/blob/AbstractBlobClientTest.java +++ b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AbstractAzureBlobClientTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.blob; +package org.apache.solr.azureblob; import com.azure.core.http.HttpClient; import com.azure.core.http.netty.NettyAsyncHttpClientBuilder; @@ -34,13 +34,13 @@ import reactor.netty.resources.ConnectionProvider; /** Abstract class for tests with Azure Blob Storage emulator. */ -public class AbstractBlobClientTest extends SolrTestCaseJ4 { +public class AbstractAzureBlobClientTest extends SolrTestCaseJ4 { protected String containerName; @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); - BlobStorageClient client; + AzureBlobStorageClient client; private static String connectionString; private EventLoopGroup eventLoopGroup; private ConnectionProvider connectionProvider; @@ -80,7 +80,7 @@ public void setUpClient() throws Exception { .buildClient(); containerName = "test-" + java.util.UUID.randomUUID(); - client = new BlobStorageClient(blobServiceClient, containerName); + client = new AzureBlobStorageClient(blobServiceClient, containerName); } /** @@ -134,7 +134,7 @@ public void tearDownClient() { } /** Simulate a connection loss on the proxy similar to S3 tests. */ - void initiateBlobConnectionLoss() throws BlobException { + void initiateBlobConnectionLoss() throws AzureBlobException { if (proxy != null) { proxy.halfClose(); } @@ -155,15 +155,15 @@ public static void afterAll() { * @param path Destination path in blob storage. * @param content Arbitrary content for the test. */ - void pushContent(String path, String content) throws BlobException { + void pushContent(String path, String content) throws AzureBlobException { pushContent(path, content.getBytes(StandardCharsets.UTF_8)); } - void pushContent(String path, byte[] content) throws BlobException { + void pushContent(String path, byte[] content) throws AzureBlobException { try (OutputStream output = client.pushStream(path)) { output.write(content); } catch (IOException e) { - throw new BlobException("Failed to write content", e); + throw new AzureBlobException("Failed to write content", e); } } diff --git a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobBackupRepositoryTest.java b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobBackupRepositoryTest.java similarity index 96% rename from solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobBackupRepositoryTest.java rename to solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobBackupRepositoryTest.java index 690808a83557..530b25c3a4c6 100644 --- a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobBackupRepositoryTest.java +++ b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobBackupRepositoryTest.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.blob; +package org.apache.solr.azureblob; -import static org.apache.solr.blob.BlobBackupRepository.BLOB_SCHEME; +import static org.apache.solr.azureblob.AzureBlobBackupRepository.BLOB_SCHEME; import java.io.IOException; import java.io.OutputStream; @@ -33,14 +33,14 @@ import org.junit.Before; import org.junit.Test; -public class BlobBackupRepositoryTest extends AbstractBlobClientTest { +public class AzureBlobBackupRepositoryTest extends AbstractAzureBlobClientTest { - private BlobBackupRepository repository; + private AzureBlobBackupRepository repository; protected static final String CONTAINER_NAME = "test-container"; protected Class getRepositoryClass() { - return BlobBackupRepository.class; + return AzureBlobBackupRepository.class; } protected BackupRepository getRepository() { @@ -62,13 +62,13 @@ public void setUp() throws Exception { // Use a repository that avoids creating its own Azure client (which leaks Netty threads) // and instead inject the pre-configured client from AbstractBlobClientTest. repository = - new BlobBackupRepository() { + new AzureBlobBackupRepository() { @Override public void init(NamedList args) { // Only capture config; avoid building a new client inside init this.config = args; // Inject the already-initialized client that uses isolated Netty resources - setClient(BlobBackupRepositoryTest.this.client); + setClient(AzureBlobBackupRepositoryTest.this.client); } }; repository.init(config); diff --git a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobIncrementalBackupTest.java b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobIncrementalBackupTest.java similarity index 98% rename from solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobIncrementalBackupTest.java rename to solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobIncrementalBackupTest.java index d3bce0b1ab88..68057dc6f8c8 100644 --- a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobIncrementalBackupTest.java +++ b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobIncrementalBackupTest.java @@ -14,12 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.blob; +package org.apache.solr.azureblob; import java.nio.charset.StandardCharsets; import org.junit.Test; -public class BlobIncrementalBackupTest extends AbstractBlobClientTest { +public class AzureBlobIncrementalBackupTest extends AbstractAzureBlobClientTest { @Test public void testIncrementalBackup() throws Exception { @@ -224,7 +224,7 @@ public void testConcurrentBackups() throws Exception { } } - private void createBackup(String backupPath, String content) throws BlobException { + private void createBackup(String backupPath, String content) throws AzureBlobException { client.createDirectory(backupPath); pushContent(backupPath + "backup-file.txt", content); } diff --git a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobIndexInputTest.java b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobIndexInputTest.java similarity index 86% rename from solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobIndexInputTest.java rename to solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobIndexInputTest.java index ccd681eab70d..7db9e286e93b 100644 --- a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobIndexInputTest.java +++ b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobIndexInputTest.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.blob; +package org.apache.solr.azureblob; import java.io.IOException; import java.nio.charset.StandardCharsets; import org.junit.Test; -public class BlobIndexInputTest extends AbstractBlobClientTest { +public class AzureBlobIndexInputTest extends AbstractAzureBlobClientTest { @Test public void testBasicIndexInput() throws Exception { @@ -31,7 +31,7 @@ public void testBasicIndexInput() throws Exception { pushContent(path, content); // Read using BlobIndexInput - try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { byte[] buffer = new byte[1024]; input.readBytes(buffer, 0, content.length()); String readContent = new String(buffer, 0, content.length(), StandardCharsets.UTF_8); @@ -48,7 +48,7 @@ public void testIndexInputSeek() throws Exception { pushContent(path, content); // Test seeking - try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { // Seek to middle of content long seekPosition = content.length() / 2; input.seek(seekPosition); @@ -71,7 +71,7 @@ public void testIndexInputLength() throws Exception { pushContent(path, content); // Test length - try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { assertEquals("Length should match", content.length(), input.length()); } } @@ -85,7 +85,7 @@ public void testIndexInputReadByte() throws Exception { pushContent(path, content); // Test reading byte by byte - try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { StringBuilder readContent = new StringBuilder(); for (int i = 0; i < content.length(); i++) { byte b = input.readByte(); @@ -104,7 +104,7 @@ public void testIndexInputReadBytes() throws Exception { pushContent(path, content); // Test reading bytes - try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { byte[] buffer = new byte[10]; StringBuilder readContent = new StringBuilder(); @@ -130,7 +130,7 @@ public void testIndexInputSeekToEnd() throws Exception { pushContent(path, content); // Test seeking to end - try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { input.seek(content.length()); // Should be at end, no more bytes to read @@ -152,7 +152,7 @@ public void testIndexInputSeekBeyondEnd() throws Exception { pushContent(path, content); // Test seeking beyond end - try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { try { input.seek(content.length() + 1); fail("Should throw IOException when seeking beyond end"); @@ -171,7 +171,7 @@ public void testIndexInputGetFilePointer() throws Exception { pushContent(path, content); // Test file pointer - try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { assertEquals("Initial position should be 0", 0, input.getFilePointer()); // Read some bytes @@ -200,7 +200,7 @@ public void testIndexInputLargeFile() throws Exception { pushContent(path, content); // Test reading large file - try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { assertEquals("Length should match", content.length(), input.length()); // Read in chunks @@ -229,7 +229,7 @@ public void testIndexInputEmptyFile() throws Exception { pushContent(path, content); // Test reading empty file - try (BlobIndexInput input = new BlobIndexInput(path, client, client.length(path))) { + try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { assertEquals("Length should be 0", 0, input.length()); assertEquals("Position should be 0", 0, input.getFilePointer()); @@ -252,7 +252,7 @@ public void testIndexInputClose() throws Exception { pushContent(path, content); // Test closing - BlobIndexInput input = new BlobIndexInput(path, client, client.length(path)); + AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path)); input.close(); // Test that operations on closed input throw exception @@ -280,7 +280,7 @@ public void testIndexInputMultipleClose() throws Exception { pushContent(path, content); // Test multiple close calls - BlobIndexInput input = new BlobIndexInput(path, client, client.length(path)); + AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path)); input.close(); input.close(); // Should not throw exception } diff --git a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobInstallShardTest.java b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobInstallShardTest.java similarity index 98% rename from solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobInstallShardTest.java rename to solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobInstallShardTest.java index e89ca6e2a402..de18f10bcaf7 100644 --- a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobInstallShardTest.java +++ b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobInstallShardTest.java @@ -14,12 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.blob; +package org.apache.solr.azureblob; import java.nio.charset.StandardCharsets; import org.junit.Test; -public class BlobInstallShardTest extends AbstractBlobClientTest { +public class AzureBlobInstallShardTest extends AbstractAzureBlobClientTest { @Test public void testInstallShard() throws Exception { diff --git a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobOutputStreamTest.java b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobOutputStreamTest.java similarity index 98% rename from solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobOutputStreamTest.java rename to solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobOutputStreamTest.java index f943264fa410..919b72d1a30d 100644 --- a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobOutputStreamTest.java +++ b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobOutputStreamTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.blob; +package org.apache.solr.azureblob; import java.io.IOException; import java.io.InputStream; @@ -22,7 +22,7 @@ import java.nio.charset.StandardCharsets; import org.junit.Test; -public class BlobOutputStreamTest extends AbstractBlobClientTest { +public class AzureBlobOutputStreamTest extends AbstractAzureBlobClientTest { @Test public void testBasicOutputStream() throws Exception { diff --git a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobPathsTest.java b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobPathsTest.java similarity index 96% rename from solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobPathsTest.java rename to solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobPathsTest.java index 7cd3606d7d6f..787038dea440 100644 --- a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobPathsTest.java +++ b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobPathsTest.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.blob; +package org.apache.solr.azureblob; import org.junit.Test; -public class BlobPathsTest extends AbstractBlobClientTest { +public class AzureBlobPathsTest extends AbstractAzureBlobClientTest { @Test public void testPathExists() throws Exception { @@ -85,7 +85,7 @@ public void testDirectoryLength() throws Exception { try { client.length(dirPath); fail("Should throw exception when getting length of directory"); - } catch (BlobException e) { + } catch (AzureBlobException e) { // Expected } } @@ -155,7 +155,7 @@ public void testListAll() throws Exception { } private void listAllRecursive(String dirPath, java.util.Set allFiles) - throws BlobException { + throws AzureBlobException { String[] files = client.listDir(dirPath); for (String file : files) { String fullPath = dirPath + file; @@ -276,7 +276,7 @@ public void testPathSanitization() throws Exception { String sanitizedPath = client.sanitizedPath(testPath); assertNotNull("Sanitized path should not be null", sanitizedPath); assertFalse("Sanitized path should not start with slash", sanitizedPath.startsWith("/")); - } catch (BlobException e) { + } catch (AzureBlobException e) { // Some paths might be invalid, which is expected } } @@ -294,7 +294,7 @@ public void testFilePathSanitization() throws Exception { String sanitizedPath = client.sanitizedFilePath(filePath); assertNotNull("Sanitized file path should not be null", sanitizedPath); assertFalse("Sanitized file path should not end with slash", sanitizedPath.endsWith("/")); - } catch (BlobException e) { + } catch (AzureBlobException e) { fail("Valid file path should not throw exception: " + filePath); } } @@ -306,7 +306,7 @@ public void testFilePathSanitization() throws Exception { try { client.sanitizedFilePath(filePath); fail("Invalid file path should throw exception: " + filePath); - } catch (BlobException e) { + } catch (AzureBlobException e) { // Expected } } @@ -324,7 +324,7 @@ public void testDirectoryPathSanitization() throws Exception { String sanitizedPath = client.sanitizedDirPath(dirPath); assertNotNull("Sanitized directory path should not be null", sanitizedPath); assertTrue("Sanitized directory path should end with slash", sanitizedPath.endsWith("/")); - } catch (BlobException e) { + } catch (AzureBlobException e) { fail("Valid directory path should not throw exception: " + dirPath); } } diff --git a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobReadWriteTest.java b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobReadWriteTest.java similarity index 98% rename from solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobReadWriteTest.java rename to solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobReadWriteTest.java index 93256ae7058c..370fe7321d29 100644 --- a/solr/modules/blob-repository/src/test/org/apache/solr/blob/BlobReadWriteTest.java +++ b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobReadWriteTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.blob; +package org.apache.solr.azureblob; import com.carrotsearch.randomizedtesting.generators.RandomBytes; import java.io.IOException; @@ -23,7 +23,7 @@ import java.nio.charset.StandardCharsets; import org.junit.Test; -public class BlobReadWriteTest extends AbstractBlobClientTest { +public class AzureBlobReadWriteTest extends AbstractAzureBlobClientTest { @Test public void testBasicReadWrite() throws Exception { @@ -269,7 +269,7 @@ public void testReadWithConnectionLoss() throws Exception { if (currentBucket != lastResetBucket && (byteCount % bytesPerException <= maxBuffer)) { try { initiateBlobConnectionLoss(); - } catch (BlobException e) { + } catch (AzureBlobException e) { throw new IOException("Failed to simulate connection loss", e); } lastResetBucket = currentBucket; diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc index 92f5980d4888..d56a458db21e 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc @@ -799,9 +799,9 @@ https://docs.aws.amazon.com/sdkref/latest/guide/settings-global.html[These optio Stores and retrieves backup files in a Microsoft Azure Blob Storage container. -This is provided via the `blob-repository` xref:configuration-guide:solr-modules.adoc[Solr Module] that needs to be enabled before use. +This is provided via the `azure-blob-repository` xref:configuration-guide:solr-modules.adoc[Solr Module] that needs to be enabled before use. -BlobBackupRepository supports four authentication methods, each suitable for different deployment scenarios: +AzureBlobBackupRepository supports four authentication methods, each suitable for different deployment scenarios: ==== Authentication Methods @@ -813,9 +813,9 @@ Ideal for local development with Azurite emulator or quick testing. [source,xml] ---- - - solr-backup - DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net + + solr-backup + DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net ---- @@ -827,10 +827,10 @@ Separates the account name from the access key, providing cleaner configuration [source,xml] ---- - - solr-backup - myaccount - mykey + + solr-backup + myaccount + mykey ---- @@ -845,10 +845,10 @@ The container must be pre-created before using a SAS token. [source,xml] ---- - - solr-backup - myaccount - sv=2024-11-04&ss=b&srt=sco&sp=rwdlacytfx&se=2025-12-31T23:59:59Z&st=2025-01-01T00:00:00Z&spr=https&sig=... + + solr-backup + myaccount + sv=2024-11-04&ss=b&srt=sco&sp=rwdlacytfx&se=2025-12-31T23:59:59Z&st=2025-01-01T00:00:00Z&spr=https&sig=... ---- @@ -865,9 +865,9 @@ For *Managed Identity* (for VMs, AKS, App Service): [source,xml] ---- - - solr-backup - https://myaccount.blob.core.windows.net + + solr-backup + https://myaccount.blob.core.windows.net ---- @@ -877,12 +877,12 @@ For *Service Principal*: [source,xml] ---- - - solr-backup - https://myaccount.blob.core.windows.net - your-tenant-id - your-client-id - your-client-secret + + solr-backup + https://myaccount.blob.core.windows.net + your-tenant-id + your-client-id + your-client-secret ---- @@ -892,9 +892,9 @@ For *Azure CLI* (development only): [source,xml] ---- - - solr-backup - https://myaccount.blob.core.windows.net + + solr-backup + https://myaccount.blob.core.windows.net ---- @@ -903,9 +903,9 @@ NOTE: When using Azure Identity, the identity must have the "Storage Blob Data C ==== Configuration Options -BlobBackupRepository accepts the following configuration options: +AzureBlobBackupRepository accepts the following configuration options: -`blob.container.name`:: +`azure.blob.container.name`:: + [%autowidth,frame=none] |=== @@ -915,7 +915,7 @@ BlobBackupRepository accepts the following configuration options: The name of the Azure Blob Storage container to use for backups. The container must exist before performing backup operations. -`blob.connection.string`:: +`azure.blob.connection.string`:: + [%autowidth,frame=none] |=== @@ -926,7 +926,7 @@ Complete Azure Storage connection string including account name, key, and endpoi Required for Connection String authentication. Mutually exclusive with other authentication methods. -`blob.account.name`:: +`azure.blob.account.name`:: + [%autowidth,frame=none] |=== @@ -936,7 +936,7 @@ Mutually exclusive with other authentication methods. Azure Storage account name. Required for Account Name + Key and SAS Token authentication methods. -`blob.account.key`:: +`azure.blob.account.key`:: + [%autowidth,frame=none] |=== @@ -947,7 +947,7 @@ Azure Storage account access key. Required for Account Name + Key authentication. Mutually exclusive with SAS token and Azure Identity. -`blob.sas.token`:: +`azure.blob.sas.token`:: + [%autowidth,frame=none] |=== @@ -959,7 +959,7 @@ Must include `srt=sco` (service, container, object) and `sp=rwdlac` permissions. The `&` characters must be XML-escaped as `&` in `solr.xml`. Mutually exclusive with account key and Azure Identity. -`blob.endpoint`:: +`azure.blob.endpoint`:: + [%autowidth,frame=none] |=== @@ -970,7 +970,7 @@ Azure Blob Storage endpoint URL in the format `https://.blob.core.windo Required for Azure Identity authentication. Can be used with other methods to override default endpoint. -`azure.tenant.id`:: +`azure.blob.tenant.id`:: + [%autowidth,frame=none] |=== @@ -980,7 +980,7 @@ Can be used with other methods to override default endpoint. Azure Active Directory tenant ID. Required for Service Principal authentication. -`azure.client.id`:: +`azure.blob.client.id`:: + [%autowidth,frame=none] |=== @@ -990,7 +990,7 @@ Required for Service Principal authentication. Azure Active Directory application (client) ID. Required for Service Principal authentication. -`azure.client.secret`:: +`azure.blob.client.secret`:: + [%autowidth,frame=none] |=== @@ -1026,9 +1026,9 @@ Configure `solr.xml` with Azurite connection string: [source,xml] ---- - - solr-backup - DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1; + + solr-backup + DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1; ---- @@ -1043,4 +1043,4 @@ Configure `solr.xml` with Azurite connection string: * *Monitor backup operations* through Azure Storage metrics and logs * *Test restore operations* regularly to ensure backup integrity -For more detailed information on Azure authentication setup, SAS token generation, and troubleshooting, refer to the module documentation in `solr/modules/blob-repository/README.md`. +For more detailed information on Azure authentication setup, SAS token generation, and troubleshooting, refer to the module documentation in `solr/modules/azure-blob-repository/README.md`. From 219f46228445e361a32bcab748b8e658c3657e8e Mon Sep 17 00:00:00 2001 From: Prateek Singhal Date: Mon, 24 Nov 2025 15:41:06 -0800 Subject: [PATCH 3/5] SOLR-17949: Switch to OkHttp and use shared HttpClient - Switch from Netty to OkHttp for better Security Manager compatibility - Use static shared HttpClient for better resource management - Fix licenses: msal4j and Azure SDK are MIT licensed - Add changelog entry - Add JFR permissions for Reactor --- .../SOLR-17949-azure-blob-repository.yml | 11 + gradle/libs.versions.toml | 4 +- solr/licenses/azure-LICENSE-ASL.txt | 206 ------------------ solr/licenses/azure-LICENSE-MIT.txt | 22 ++ solr/licenses/azure-NOTICE.txt | 12 - solr/licenses/azure-core-1.52.0.jar.sha1 | 1 - solr/licenses/azure-core-1.57.0.jar.sha1 | 1 + .../azure-core-http-netty-1.15.4.jar.sha1 | 1 - .../azure-core-http-okhttp-1.13.2.jar.sha1 | 1 + solr/licenses/azure-json-1.3.0.jar.sha1 | 1 - solr/licenses/azure-json-1.5.0.jar.sha1 | 1 + solr/licenses/azure-xml-1.1.0.jar.sha1 | 1 - solr/licenses/azure-xml-1.2.0.jar.sha1 | 1 + solr/licenses/msal4j-LICENSE-ASL.txt | 206 ------------------ solr/licenses/msal4j-LICENSE-MIT.txt | 22 ++ solr/licenses/msal4j-NOTICE.txt | 12 - .../netty-buffer-4.1.110.Final.jar.sha1 | 1 - .../netty-codec-4.1.110.Final.jar.sha1 | 1 - .../netty-codec-dns-4.1.110.Final.jar.sha1 | 1 - .../netty-codec-http-4.1.110.Final.jar.sha1 | 1 - .../netty-codec-http2-4.1.110.Final.jar.sha1 | 1 - .../netty-codec-socks-4.1.110.Final.jar.sha1 | 1 - .../netty-common-4.1.110.Final.jar.sha1 | 1 - .../netty-handler-4.1.110.Final.jar.sha1 | 1 - ...netty-handler-proxy-4.1.110.Final.jar.sha1 | 1 - .../netty-resolver-4.1.110.Final.jar.sha1 | 1 - .../netty-resolver-dns-4.1.110.Final.jar.sha1 | 1 - ...r-dns-classes-macos-4.1.110.Final.jar.sha1 | 1 - ...ve-macos-4.1.110.Final-osx-x86_64.jar.sha1 | 1 - ...ive-boringssl-static-2.0.65.Final.jar.sha1 | 1 - ...tty-tcnative-classes-2.0.65.Final.jar.sha1 | 1 - .../netty-transport-4.1.110.Final.jar.sha1 | 1 - ...sport-classes-epoll-4.1.110.Final.jar.sha1 | 1 - ...port-classes-kqueue-4.1.110.Final.jar.sha1 | 1 - ...-epoll-4.1.110.Final-linux-x86_64.jar.sha1 | 1 - ...e-kqueue-4.1.110.Final-osx-x86_64.jar.sha1 | 1 - ...-native-unix-common-4.1.110.Final.jar.sha1 | 1 - solr/licenses/okio-jvm-3.16.0.jar.sha1 | 1 + solr/licenses/reactor-core-3.4.38.jar.sha1 | 1 - solr/licenses/reactor-core-3.7.11.jar.sha1 | 1 + .../reactor-netty-core-1.0.45.jar.sha1 | 1 - .../reactor-netty-http-1.0.45.jar.sha1 | 1 - .../azure-blob-repository/build.gradle | 24 +- .../azureblob/AzureBlobStorageClient.java | 11 +- solr/server/etc/security.policy | 3 + 45 files changed, 92 insertions(+), 475 deletions(-) create mode 100644 changelog/unreleased/SOLR-17949-azure-blob-repository.yml delete mode 100644 solr/licenses/azure-LICENSE-ASL.txt create mode 100644 solr/licenses/azure-LICENSE-MIT.txt delete mode 100644 solr/licenses/azure-NOTICE.txt delete mode 100644 solr/licenses/azure-core-1.52.0.jar.sha1 create mode 100644 solr/licenses/azure-core-1.57.0.jar.sha1 delete mode 100644 solr/licenses/azure-core-http-netty-1.15.4.jar.sha1 create mode 100644 solr/licenses/azure-core-http-okhttp-1.13.2.jar.sha1 delete mode 100644 solr/licenses/azure-json-1.3.0.jar.sha1 create mode 100644 solr/licenses/azure-json-1.5.0.jar.sha1 delete mode 100644 solr/licenses/azure-xml-1.1.0.jar.sha1 create mode 100644 solr/licenses/azure-xml-1.2.0.jar.sha1 delete mode 100644 solr/licenses/msal4j-LICENSE-ASL.txt create mode 100644 solr/licenses/msal4j-LICENSE-MIT.txt delete mode 100644 solr/licenses/msal4j-NOTICE.txt delete mode 100644 solr/licenses/netty-buffer-4.1.110.Final.jar.sha1 delete mode 100644 solr/licenses/netty-codec-4.1.110.Final.jar.sha1 delete mode 100644 solr/licenses/netty-codec-dns-4.1.110.Final.jar.sha1 delete mode 100644 solr/licenses/netty-codec-http-4.1.110.Final.jar.sha1 delete mode 100644 solr/licenses/netty-codec-http2-4.1.110.Final.jar.sha1 delete mode 100644 solr/licenses/netty-codec-socks-4.1.110.Final.jar.sha1 delete mode 100644 solr/licenses/netty-common-4.1.110.Final.jar.sha1 delete mode 100644 solr/licenses/netty-handler-4.1.110.Final.jar.sha1 delete mode 100644 solr/licenses/netty-handler-proxy-4.1.110.Final.jar.sha1 delete mode 100644 solr/licenses/netty-resolver-4.1.110.Final.jar.sha1 delete mode 100644 solr/licenses/netty-resolver-dns-4.1.110.Final.jar.sha1 delete mode 100644 solr/licenses/netty-resolver-dns-classes-macos-4.1.110.Final.jar.sha1 delete mode 100644 solr/licenses/netty-resolver-dns-native-macos-4.1.110.Final-osx-x86_64.jar.sha1 delete mode 100644 solr/licenses/netty-tcnative-boringssl-static-2.0.65.Final.jar.sha1 delete mode 100644 solr/licenses/netty-tcnative-classes-2.0.65.Final.jar.sha1 delete mode 100644 solr/licenses/netty-transport-4.1.110.Final.jar.sha1 delete mode 100644 solr/licenses/netty-transport-classes-epoll-4.1.110.Final.jar.sha1 delete mode 100644 solr/licenses/netty-transport-classes-kqueue-4.1.110.Final.jar.sha1 delete mode 100644 solr/licenses/netty-transport-native-epoll-4.1.110.Final-linux-x86_64.jar.sha1 delete mode 100644 solr/licenses/netty-transport-native-kqueue-4.1.110.Final-osx-x86_64.jar.sha1 delete mode 100644 solr/licenses/netty-transport-native-unix-common-4.1.110.Final.jar.sha1 create mode 100644 solr/licenses/okio-jvm-3.16.0.jar.sha1 delete mode 100644 solr/licenses/reactor-core-3.4.38.jar.sha1 create mode 100644 solr/licenses/reactor-core-3.7.11.jar.sha1 delete mode 100644 solr/licenses/reactor-netty-core-1.0.45.jar.sha1 delete mode 100644 solr/licenses/reactor-netty-http-1.0.45.jar.sha1 diff --git a/changelog/unreleased/SOLR-17949-azure-blob-repository.yml b/changelog/unreleased/SOLR-17949-azure-blob-repository.yml new file mode 100644 index 000000000000..6ec39bd703dd --- /dev/null +++ b/changelog/unreleased/SOLR-17949-azure-blob-repository.yml @@ -0,0 +1,11 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Add Azure Blob Storage backup repository module +type: added +authors: + - name: Prateek Singhal +description: | + Added AzureBlobBackupRepository module for backing up and restoring Solr collections to Azure Blob Storage. + Supports multiple authentication methods: connection string, account name + key, SAS token, and Azure Identity (service principal, managed identity). +links: + - name: SOLR-17949 + url: https://issues.apache.org/jira/browse/SOLR-17949 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f7627548e280..9fc068e7452a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -51,7 +51,7 @@ asciidoctor-mathjax = "0.0.9" # @keep Asciidoctor tabs version used in ref-guide asciidoctor-tabs = "1.0.0-beta.6" azure-core = "1.52.0" -azure-core-http-netty = "1.15.4" +azure-core-http-okhttp = "1.13.2" azure-identity = "1.12.0" azure-storage = "12.25.0" # @keep bats-assert (node) version used in packaging @@ -309,7 +309,7 @@ apache-zookeeper-zookeeper = { module = "org.apache.zookeeper:zookeeper", versio apiguardian-api = { module = "org.apiguardian:apiguardian-api", version.ref = "apiguardian" } aqute-bnd-annotation = { module = "biz.aQute.bnd:biz.aQute.bnd.annotation", version.ref = "aqute-bnd" } azure-core = { module = "com.azure:azure-core", version.ref = "azure-core" } -azure-core-http-netty = { module = "com.azure:azure-core-http-netty", version.ref = "azure-core-http-netty" } +azure-core-http-okhttp = { module = "com.azure:azure-core-http-okhttp", version.ref = "azure-core-http-okhttp" } azure-identity = { module = "com.azure:azure-identity", version.ref = "azure-identity" } azure-storage-blob = { module = "com.azure:azure-storage-blob", version.ref = "azure-storage" } bc-jose4j = { module = "org.bitbucket.b_c:jose4j", version.ref = "bc-jose4j" } diff --git a/solr/licenses/azure-LICENSE-ASL.txt b/solr/licenses/azure-LICENSE-ASL.txt deleted file mode 100644 index 1eef70a9b9f4..000000000000 --- a/solr/licenses/azure-LICENSE-ASL.txt +++ /dev/null @@ -1,206 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - Note: Other license terms may apply to certain, identified software files contained within or distributed - with the accompanying software if such terms are included in the directory containing the accompanying software. - Such other license terms will then apply in lieu of the terms of the software license above. diff --git a/solr/licenses/azure-LICENSE-MIT.txt b/solr/licenses/azure-LICENSE-MIT.txt new file mode 100644 index 000000000000..b8b569d7746d --- /dev/null +++ b/solr/licenses/azure-LICENSE-MIT.txt @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 Microsoft + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/solr/licenses/azure-NOTICE.txt b/solr/licenses/azure-NOTICE.txt deleted file mode 100644 index 2831e601401c..000000000000 --- a/solr/licenses/azure-NOTICE.txt +++ /dev/null @@ -1,12 +0,0 @@ -Azure SDK for Java -Copyright (c) Microsoft Corporation. All rights reserved. - -This product includes software developed by -Microsoft Corporation (https://github.com/Azure/azure-sdk-for-java). - -Licensed under the MIT License. - -********************** -THIRD PARTY COMPONENTS -********************** -This software may include third party software subject to the following copyrights: diff --git a/solr/licenses/azure-core-1.52.0.jar.sha1 b/solr/licenses/azure-core-1.52.0.jar.sha1 deleted file mode 100644 index e0d4f012e79d..000000000000 --- a/solr/licenses/azure-core-1.52.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -43bd4ad76e6772d24c545635b48e0ed4d0e511f2 diff --git a/solr/licenses/azure-core-1.57.0.jar.sha1 b/solr/licenses/azure-core-1.57.0.jar.sha1 new file mode 100644 index 000000000000..61da6e275e4e --- /dev/null +++ b/solr/licenses/azure-core-1.57.0.jar.sha1 @@ -0,0 +1 @@ +4fe5978491bb9a305b98dc5456a138ad7ba0f250 diff --git a/solr/licenses/azure-core-http-netty-1.15.4.jar.sha1 b/solr/licenses/azure-core-http-netty-1.15.4.jar.sha1 deleted file mode 100644 index 614ec2b5b116..000000000000 --- a/solr/licenses/azure-core-http-netty-1.15.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -489a38c9e6efb5ce01fbd276d8cb6c0e89000459 diff --git a/solr/licenses/azure-core-http-okhttp-1.13.2.jar.sha1 b/solr/licenses/azure-core-http-okhttp-1.13.2.jar.sha1 new file mode 100644 index 000000000000..c7a3ae4a128a --- /dev/null +++ b/solr/licenses/azure-core-http-okhttp-1.13.2.jar.sha1 @@ -0,0 +1 @@ +fd743d404300f134a2740c6d2ec8dbf9ebafcf04 diff --git a/solr/licenses/azure-json-1.3.0.jar.sha1 b/solr/licenses/azure-json-1.3.0.jar.sha1 deleted file mode 100644 index 47daa904564b..000000000000 --- a/solr/licenses/azure-json-1.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -11b6a0708e9d6c90a1a76574c7720edce47dacc1 diff --git a/solr/licenses/azure-json-1.5.0.jar.sha1 b/solr/licenses/azure-json-1.5.0.jar.sha1 new file mode 100644 index 000000000000..06c3f5e6cdc8 --- /dev/null +++ b/solr/licenses/azure-json-1.5.0.jar.sha1 @@ -0,0 +1 @@ +d12cf1a1d31ca75b27a5bbe0fbcf5ad73b7471b5 diff --git a/solr/licenses/azure-xml-1.1.0.jar.sha1 b/solr/licenses/azure-xml-1.1.0.jar.sha1 deleted file mode 100644 index 1224ee5783bb..000000000000 --- a/solr/licenses/azure-xml-1.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8218a00c07f9f66d5dc7ae2ba613da6890867497 diff --git a/solr/licenses/azure-xml-1.2.0.jar.sha1 b/solr/licenses/azure-xml-1.2.0.jar.sha1 new file mode 100644 index 000000000000..75c0d7a6e8b9 --- /dev/null +++ b/solr/licenses/azure-xml-1.2.0.jar.sha1 @@ -0,0 +1 @@ +05a811882dc4eba119c7d1f0fc65acf39eaf417c diff --git a/solr/licenses/msal4j-LICENSE-ASL.txt b/solr/licenses/msal4j-LICENSE-ASL.txt deleted file mode 100644 index 1eef70a9b9f4..000000000000 --- a/solr/licenses/msal4j-LICENSE-ASL.txt +++ /dev/null @@ -1,206 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - Note: Other license terms may apply to certain, identified software files contained within or distributed - with the accompanying software if such terms are included in the directory containing the accompanying software. - Such other license terms will then apply in lieu of the terms of the software license above. diff --git a/solr/licenses/msal4j-LICENSE-MIT.txt b/solr/licenses/msal4j-LICENSE-MIT.txt new file mode 100644 index 000000000000..ad22b888b221 --- /dev/null +++ b/solr/licenses/msal4j-LICENSE-MIT.txt @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) Microsoft Corporation. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE + diff --git a/solr/licenses/msal4j-NOTICE.txt b/solr/licenses/msal4j-NOTICE.txt deleted file mode 100644 index cf861d18eea7..000000000000 --- a/solr/licenses/msal4j-NOTICE.txt +++ /dev/null @@ -1,12 +0,0 @@ -Microsoft Authentication Library for Java (MSAL4J) -Copyright (c) Microsoft Corporation. All rights reserved. - -This product includes software developed by -Microsoft Corporation (https://github.com/AzureAD/microsoft-authentication-library-for-java). - -Licensed under the MIT License. - -********************** -THIRD PARTY COMPONENTS -********************** -This software may include third party software subject to the following copyrights: diff --git a/solr/licenses/netty-buffer-4.1.110.Final.jar.sha1 b/solr/licenses/netty-buffer-4.1.110.Final.jar.sha1 deleted file mode 100644 index bb8c75abbcdf..000000000000 --- a/solr/licenses/netty-buffer-4.1.110.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3d918a9ee057d995c362902b54634fc307132aac diff --git a/solr/licenses/netty-codec-4.1.110.Final.jar.sha1 b/solr/licenses/netty-codec-4.1.110.Final.jar.sha1 deleted file mode 100644 index a41772233da8..000000000000 --- a/solr/licenses/netty-codec-4.1.110.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f1fa43b03e93ab88e805b6a4e3e83780c80b47d2 diff --git a/solr/licenses/netty-codec-dns-4.1.110.Final.jar.sha1 b/solr/licenses/netty-codec-dns-4.1.110.Final.jar.sha1 deleted file mode 100644 index 0cb6e0d23a43..000000000000 --- a/solr/licenses/netty-codec-dns-4.1.110.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -381c5bf8b7570c163fa7893a26d02b7ac36ff6eb diff --git a/solr/licenses/netty-codec-http-4.1.110.Final.jar.sha1 b/solr/licenses/netty-codec-http-4.1.110.Final.jar.sha1 deleted file mode 100644 index 00574566267e..000000000000 --- a/solr/licenses/netty-codec-http-4.1.110.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9d05cd927209ea25bbf342962c00b8e5a828c2a4 diff --git a/solr/licenses/netty-codec-http2-4.1.110.Final.jar.sha1 b/solr/licenses/netty-codec-http2-4.1.110.Final.jar.sha1 deleted file mode 100644 index 27bcd9e7dc43..000000000000 --- a/solr/licenses/netty-codec-http2-4.1.110.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e0849843eb5b1c036b12551baca98a9f7ff847a0 diff --git a/solr/licenses/netty-codec-socks-4.1.110.Final.jar.sha1 b/solr/licenses/netty-codec-socks-4.1.110.Final.jar.sha1 deleted file mode 100644 index 0c7f8c8d5411..000000000000 --- a/solr/licenses/netty-codec-socks-4.1.110.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4d54c8d5b95b14756043efb59b8c3e62ec67aa43 diff --git a/solr/licenses/netty-common-4.1.110.Final.jar.sha1 b/solr/licenses/netty-common-4.1.110.Final.jar.sha1 deleted file mode 100644 index 588f41bee630..000000000000 --- a/solr/licenses/netty-common-4.1.110.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ec361e7e025c029be50c55c8480080cabcbc01e7 diff --git a/solr/licenses/netty-handler-4.1.110.Final.jar.sha1 b/solr/licenses/netty-handler-4.1.110.Final.jar.sha1 deleted file mode 100644 index 8946e71e1483..000000000000 --- a/solr/licenses/netty-handler-4.1.110.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -168db749c22652ee7fed1ebf7ec46ce856d75e51 diff --git a/solr/licenses/netty-handler-proxy-4.1.110.Final.jar.sha1 b/solr/licenses/netty-handler-proxy-4.1.110.Final.jar.sha1 deleted file mode 100644 index 33ded80b73e3..000000000000 --- a/solr/licenses/netty-handler-proxy-4.1.110.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b7fb401dd47c79e6b99f2319ac3b561c50c31c30 diff --git a/solr/licenses/netty-resolver-4.1.110.Final.jar.sha1 b/solr/licenses/netty-resolver-4.1.110.Final.jar.sha1 deleted file mode 100644 index 50c7d92a43e8..000000000000 --- a/solr/licenses/netty-resolver-4.1.110.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -66c15921104cda0159b34e316541bc765dfaf3c0 diff --git a/solr/licenses/netty-resolver-dns-4.1.110.Final.jar.sha1 b/solr/licenses/netty-resolver-dns-4.1.110.Final.jar.sha1 deleted file mode 100644 index 1eb243870cf1..000000000000 --- a/solr/licenses/netty-resolver-dns-4.1.110.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3e687cdc4ecdbbad07508a11b715bdf95fa20939 diff --git a/solr/licenses/netty-resolver-dns-classes-macos-4.1.110.Final.jar.sha1 b/solr/licenses/netty-resolver-dns-classes-macos-4.1.110.Final.jar.sha1 deleted file mode 100644 index 2be06f13e0c7..000000000000 --- a/solr/licenses/netty-resolver-dns-classes-macos-4.1.110.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4be9633daf46657dd94851ce44adaea14a2faa7e diff --git a/solr/licenses/netty-resolver-dns-native-macos-4.1.110.Final-osx-x86_64.jar.sha1 b/solr/licenses/netty-resolver-dns-native-macos-4.1.110.Final-osx-x86_64.jar.sha1 deleted file mode 100644 index 63f71cb28d3e..000000000000 --- a/solr/licenses/netty-resolver-dns-native-macos-4.1.110.Final-osx-x86_64.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6376510bb8a8c755a1f0af1d27c2902a1c84f58c diff --git a/solr/licenses/netty-tcnative-boringssl-static-2.0.65.Final.jar.sha1 b/solr/licenses/netty-tcnative-boringssl-static-2.0.65.Final.jar.sha1 deleted file mode 100644 index c083dbe75686..000000000000 --- a/solr/licenses/netty-tcnative-boringssl-static-2.0.65.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b31c6944d9cfd596b6c25fe17e36780bfa2d7473 diff --git a/solr/licenses/netty-tcnative-classes-2.0.65.Final.jar.sha1 b/solr/licenses/netty-tcnative-classes-2.0.65.Final.jar.sha1 deleted file mode 100644 index f95844b2b89f..000000000000 --- a/solr/licenses/netty-tcnative-classes-2.0.65.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3a7aecd4bcaf75c7b0b02c26ea6ceacf3e8f5f4d diff --git a/solr/licenses/netty-transport-4.1.110.Final.jar.sha1 b/solr/licenses/netty-transport-4.1.110.Final.jar.sha1 deleted file mode 100644 index 29293a1ab6d5..000000000000 --- a/solr/licenses/netty-transport-4.1.110.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b91f04c39ac14d6a29d07184ef305953ee6e0348 diff --git a/solr/licenses/netty-transport-classes-epoll-4.1.110.Final.jar.sha1 b/solr/licenses/netty-transport-classes-epoll-4.1.110.Final.jar.sha1 deleted file mode 100644 index 75620db21e76..000000000000 --- a/solr/licenses/netty-transport-classes-epoll-4.1.110.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3ca1cff0bf82bfd38e89f6946e54f24cbb3424a2 diff --git a/solr/licenses/netty-transport-classes-kqueue-4.1.110.Final.jar.sha1 b/solr/licenses/netty-transport-classes-kqueue-4.1.110.Final.jar.sha1 deleted file mode 100644 index db1bf439ad40..000000000000 --- a/solr/licenses/netty-transport-classes-kqueue-4.1.110.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ae6037a535779ba61e316551cc6245eb1707ff7a diff --git a/solr/licenses/netty-transport-native-epoll-4.1.110.Final-linux-x86_64.jar.sha1 b/solr/licenses/netty-transport-native-epoll-4.1.110.Final-linux-x86_64.jar.sha1 deleted file mode 100644 index 5d194b1e7dbf..000000000000 --- a/solr/licenses/netty-transport-native-epoll-4.1.110.Final-linux-x86_64.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -72b74a82d22e215d1f2573c040078e0afff519af diff --git a/solr/licenses/netty-transport-native-kqueue-4.1.110.Final-osx-x86_64.jar.sha1 b/solr/licenses/netty-transport-native-kqueue-4.1.110.Final-osx-x86_64.jar.sha1 deleted file mode 100644 index 9821c7805fc0..000000000000 --- a/solr/licenses/netty-transport-native-kqueue-4.1.110.Final-osx-x86_64.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d153b25a358851f15acdd70aeb43e6830500a6be diff --git a/solr/licenses/netty-transport-native-unix-common-4.1.110.Final.jar.sha1 b/solr/licenses/netty-transport-native-unix-common-4.1.110.Final.jar.sha1 deleted file mode 100644 index 8e0a7bd52bc9..000000000000 --- a/solr/licenses/netty-transport-native-unix-common-4.1.110.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a7096e7c0a25a983647909d7513f5d4943d589c0 diff --git a/solr/licenses/okio-jvm-3.16.0.jar.sha1 b/solr/licenses/okio-jvm-3.16.0.jar.sha1 new file mode 100644 index 000000000000..38844b241316 --- /dev/null +++ b/solr/licenses/okio-jvm-3.16.0.jar.sha1 @@ -0,0 +1 @@ +60375cdf2fd0ed2a1dcd6db787095f732a31ff10 diff --git a/solr/licenses/reactor-core-3.4.38.jar.sha1 b/solr/licenses/reactor-core-3.4.38.jar.sha1 deleted file mode 100644 index 1ca673ac48c5..000000000000 --- a/solr/licenses/reactor-core-3.4.38.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -94178266e36e6de6338a1c180efaddcff0251002 diff --git a/solr/licenses/reactor-core-3.7.11.jar.sha1 b/solr/licenses/reactor-core-3.7.11.jar.sha1 new file mode 100644 index 000000000000..cae3d145d817 --- /dev/null +++ b/solr/licenses/reactor-core-3.7.11.jar.sha1 @@ -0,0 +1 @@ +8ac8ee9da2424c81c029f8c361e34838f77a1b78 diff --git a/solr/licenses/reactor-netty-core-1.0.45.jar.sha1 b/solr/licenses/reactor-netty-core-1.0.45.jar.sha1 deleted file mode 100644 index e241697b42e3..000000000000 --- a/solr/licenses/reactor-netty-core-1.0.45.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -42aea422b0551b1db4dd4eddf598ccddd5408a4e diff --git a/solr/licenses/reactor-netty-http-1.0.45.jar.sha1 b/solr/licenses/reactor-netty-http-1.0.45.jar.sha1 deleted file mode 100644 index 061f41d113ae..000000000000 --- a/solr/licenses/reactor-netty-http-1.0.45.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f24886830010329239a2f10f19727ea420898fba diff --git a/solr/modules/azure-blob-repository/build.gradle b/solr/modules/azure-blob-repository/build.gradle index 8e63a84475b3..ec9597362e31 100644 --- a/solr/modules/azure-blob-repository/build.gradle +++ b/solr/modules/azure-blob-repository/build.gradle @@ -20,8 +20,6 @@ apply plugin: 'java-library' description = 'Azure Blob Storage Repository' dependencies { - implementation enforcedPlatform("io.netty:netty-bom:4.1.110.Final") - testImplementation enforcedPlatform("io.netty:netty-bom:4.1.110.Final") implementation platform(project(':platform')) api(project(':solr:core')) implementation project(':solr:solrj') @@ -29,10 +27,19 @@ dependencies { implementation libs.apache.lucene.core // Azure Storage SDK dependencies - implementation libs.azure.storage.blob - implementation libs.azure.identity - implementation libs.azure.core - implementation 'com.azure:azure-storage-common:12.25.0' + implementation(libs.azure.storage.blob) { + exclude group: 'com.azure', module: 'azure-core-http-netty' + } + implementation(libs.azure.identity) { + exclude group: 'com.azure', module: 'azure-core-http-netty' + } + implementation(libs.azure.core) { + exclude group: 'com.azure', module: 'azure-core-http-netty' + } + implementation libs.azure.core.http.okhttp + implementation('com.azure:azure-storage-common:12.25.0') { + exclude group: 'com.azure', module: 'azure-core-http-netty' + } implementation libs.google.guava implementation libs.slf4j.api @@ -44,8 +51,9 @@ dependencies { testImplementation libs.junit.junit testImplementation libs.commonsio.commonsio + // OkHttp for test client management + testImplementation libs.azure.core.http.okhttp + // Explicit transitive test dependencies for dependency analyzer testImplementation 'com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.3' - testImplementation 'io.netty:netty-common:4.1.110.Final' - testImplementation 'io.netty:netty-transport:4.1.110.Final' } \ No newline at end of file diff --git a/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobStorageClient.java b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobStorageClient.java index 62606d288adb..217ff2975781 100644 --- a/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobStorageClient.java +++ b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobStorageClient.java @@ -51,6 +51,14 @@ public class AzureBlobStorageClient { static final String BLOB_FILE_PATH_DELIMITER = "/"; + /** + * Shared HttpClient instance for all Azure Blob Storage operations. OkHttp recommends reusing a + * single OkHttpClient instance as it maintains connection pools and thread pools that are + * expensive to create. This also prevents thread leaks in tests by using shared global threads. + */ + private static final com.azure.core.http.HttpClient SHARED_HTTP_CLIENT = + new com.azure.core.http.okhttp.OkHttpAsyncHttpClientBuilder().build(); + private final BlobContainerClient containerClient; AzureBlobStorageClient( @@ -99,7 +107,8 @@ private static BlobServiceClient createInternalClient( String clientSecret) { BlobServiceClientBuilder builder = new BlobServiceClientBuilder(); - // Use default HTTP client (Netty) as provided by azure-core-http-netty + // Use shared OkHttp client for better resource management + builder.httpClient(SHARED_HTTP_CLIENT); if (StrUtils.isNotNullOrEmpty(connectionString)) { builder.connectionString(connectionString); diff --git a/solr/server/etc/security.policy b/solr/server/etc/security.policy index bc95bc46fae4..c9da789e1028 100644 --- a/solr/server/etc/security.policy +++ b/solr/server/etc/security.policy @@ -222,6 +222,9 @@ grant { }; // Permissions for OTEL Runtime Java 17 telemetry and metrics +// Also needed for Reactor (used by Azure SDK with OkHttp) grant { permission jdk.jfr.FlightRecorderPermission "accessFlightRecorder"; + permission jdk.jfr.FlightRecorderPermission "registerEvent"; + permission java.lang.RuntimePermission "accessClassInPackage.jdk.jfr.internal.event"; }; From 428ca18a2f13f98726cd43df32193d2906744786 Mon Sep 17 00:00:00 2001 From: Prateek Singhal Date: Mon, 24 Nov 2025 15:41:22 -0800 Subject: [PATCH 4/5] SOLR-17949: Fix CI test failures and update documentation - Use Testcontainers (Azurite) for integration tests to avoid hardcoded ports and external dependencies - Disable Security Manager for Azure Blob tests to support Testcontainers (similar to extraction module) - Fix OkHttp compilation error by adding explicit testImplementation - Update documentation: BlobBackupRepository -> AzureBlobBackupRepository --- solr/modules/azure-blob-repository/README.md | 464 ++---------------- .../azure-blob-repository/build.gradle | 10 + .../azureblob/AzureBlobBackupRepository.java | 23 +- .../AzureBlobBackupRepositoryConfig.java | 2 - .../solr/azureblob/AzureBlobIndexInput.java | 7 +- .../solr/azureblob/AzureBlobOutputStream.java | 22 +- .../azureblob/AzureBlobStorageClient.java | 66 +-- .../apache/solr/azureblob/package-info.java | 30 +- .../AbstractAzureBlobClientTest.java | 168 ++++--- .../AzureBlobBackupRepositoryTest.java | 36 +- .../AzureBlobIncrementalBackupTest.java | 34 +- .../azureblob/AzureBlobIndexInputTest.java | 78 +-- .../azureblob/AzureBlobInstallShardTest.java | 69 +-- .../azureblob/AzureBlobOutputStreamTest.java | 35 +- .../solr/azureblob/AzureBlobPathsTest.java | 83 +--- .../azureblob/AzureBlobReadWriteTest.java | 49 +- .../pages/backup-restore.adoc | 169 +------ 17 files changed, 247 insertions(+), 1098 deletions(-) diff --git a/solr/modules/azure-blob-repository/README.md b/solr/modules/azure-blob-repository/README.md index 5c7b573d14cf..1a4e0accca71 100644 --- a/solr/modules/azure-blob-repository/README.md +++ b/solr/modules/azure-blob-repository/README.md @@ -15,475 +15,87 @@ limitations under the License. --> -Apache Solr - Azure Blob Storage Repository -=========================================== +# Apache Solr Azure Blob Storage Backup Repository -This Azure Blob Storage repository is a backup repository implementation designed to provide backup/restore functionality to Azure Blob Storage. - -## Quick Start - -**Choose your authentication method:** - -- 🚀 **Local Development?** → Use **Connection String** (simplest) -- 🔐 **Production on Azure VM/AKS?** → Use **Managed Identity** (most secure) -- 🏢 **Production elsewhere?** → Use **Service Principal** or **SAS Token** -- 🧪 **Testing?** → Use **Azure CLI** (no config changes) - -**Prerequisites:** -- Azure Storage Account with a blob container -- Container must already exist (e.g., `solr-backup`) -- Solr azure-blob-repository module enabled -- Network access to Azure Blob Storage (HTTPS port 443) +A backup repository implementation for storing Solr backups in Azure Blob Storage. ## Prerequisites -Before configuring authentication, ensure you have: - -1. **Azure Storage Account** - Created and accessible -2. **Blob Container** - Must already exist in your storage account - ```bash - # Create container using Azure CLI - az storage container create \ - --name solr-backup \ - --account-name YOUR_ACCOUNT_NAME - ``` -3. **Solr Module** - Enable azure-blob-repository module: - ```bash - export SOLR_MODULES=azure-blob-repository - ./bin/solr start - ``` -4. **Network Access** - Solr can reach Azure Blob Storage (HTTPS port 443) - -Optional (depending on authentication method): -- **Azure CLI** installed and configured (`az login`) -- **RBAC Permissions** for Azure Identity methods -- **SAS Token** or **Account Keys** from Azure Portal - -## Authentication Options - -The Azure Blob Storage backup repository supports four authentication methods. Choose the one that best fits your security requirements and deployment environment. - -### 1. Connection String - -The simplest authentication method using a full connection string. - -#### Configuration in solr.xml: - -```xml - - - - YOUR_CONTAINER_NAME - - DefaultEndpointsProtocol=https;AccountName=YOUR_ACCOUNT_NAME;AccountKey=YOUR_ACCOUNT_KEY;EndpointSuffix=core.windows.net - - - -``` - -**Note:** This method is simple but exposes the account key in configuration. Not recommended for production environments. - -### 2. Account Name + Key - -Separates the account credentials from the endpoint configuration. - -#### Configuration in solr.xml: - -```xml - - - - YOUR_CONTAINER_NAME - https://YOUR_ACCOUNT_NAME.blob.core.windows.net - YOUR_ACCOUNT_NAME - YOUR_ACCOUNT_KEY - - -``` - -**Note:** Similar to connection string, this exposes the account key. Use with caution in production. - -### 3. SAS Token (Recommended for Production) - -**Important:** The SAS token must be configured with proper permissions to work correctly. - -#### Required SAS Token Configuration: -- **Allowed services:** Blob -- **Allowed resource types:** Service, Container, Object (`srt=sco`) -- **Allowed permissions:** Read, Write, Delete, List, Add, Create (`sp=rwdlac` minimum) -- **Protocol:** HTTPS only -- **Expiry:** Set appropriate expiration time (e.g., 1 year) - -#### Generating SAS Token (Azure Portal): -1. Navigate to your Storage Account -2. Click "Shared access signature" (left menu under "Security + networking") -3. Configure: - - Allowed services: ☑ Blob - - Allowed resource types: ☑ Service, ☑ Container, ☑ Object - - Allowed permissions: ☑ Read, ☑ Write, ☑ Delete, ☑ List, ☑ Add, ☑ Create - - Start/Expiry time: Set your desired validity period - - Allowed protocols: HTTPS only -4. Click "Generate SAS and connection string" -5. Copy the **SAS token** (remove the leading `?` if present) +- Azure Storage Account with a blob container (must already exist) +- Network access to Azure Blob Storage (HTTPS port 443) -#### Generating SAS Token (Azure CLI): +Enable the module: ```bash -az storage account generate-sas \ - --account-name YOUR_ACCOUNT_NAME \ - --services b \ - --resource-types sco \ - --permissions rwdlac \ - --expiry 2026-12-31T23:59:59Z \ - --https-only \ - --output tsv +export SOLR_MODULES=azure-blob-repository ``` -#### Configuration in solr.xml: - -```xml - - - - YOUR_CONTAINER_NAME - https://YOUR_ACCOUNT_NAME.blob.core.windows.net - sv=2024-11-04&ss=b&srt=sco&sp=rwdlac&se=2026-12-31T23:59:59Z&st=2025-01-01T00:00:00Z&spr=https&sig=YOUR_SIGNATURE - - -``` - -**Note:** In XML, `&` characters in the SAS token must be escaped as `&`. The container must already exist in Azure Blob Storage before using it with Solr. - -#### Why SAS Token? -- ✅ Time-limited access (automatically expires) -- ✅ Scoped permissions (can restrict to specific operations) -- ✅ Revocable without rotating account keys -- ✅ No account key exposure in configuration -- ✅ Can restrict to specific IP addresses - -### 4. Azure Identity (Best for Production) - -Uses Azure Active Directory (Entra ID) for authentication. Provides enterprise-grade security with **no credentials in configuration files**. - -Azure Identity supports three authentication methods: -- **Azure CLI** - For local development -- **Service Principal** - For automation and CI/CD -- **Managed Identity** - For Azure VMs/AKS (no credentials needed) - ---- +## Configuration -#### Option A: Azure CLI (Local Development) - -Best for local development and testing. Uses your Azure login credentials. - -**Prerequisites:** -- Azure CLI installed and logged in (`az login`) -- User account has "Storage Blob Data Contributor" role - -**Grant permissions:** -```bash -# Get your user's Object ID -USER_OBJECT_ID=$(az ad signed-in-user show --query id -o tsv) - -# Grant Storage Blob Data Contributor role -az role assignment create \ - --role "Storage Blob Data Contributor" \ - --assignee $USER_OBJECT_ID \ - --scope /subscriptions/YOUR_SUBSCRIPTION_ID/resourceGroups/YOUR_RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts/YOUR_ACCOUNT_NAME -``` - -**Configuration in solr.xml:** +Add to `solr.xml`: ```xml - - - YOUR_CONTAINER_NAME - https://YOUR_ACCOUNT_NAME.blob.core.windows.net - - + + YOUR_CONTAINER_NAME + + ``` ---- +## Authentication Methods -#### Option B: Service Principal (Automation/CI-CD) - -Best for automation, CI/CD pipelines, and production deployments outside of Azure. - -**Create Service Principal:** -```bash -az ad sp create-for-rbac \ - --name "solr-backup-sp" \ - --role "Storage Blob Data Contributor" \ - --scopes /subscriptions/YOUR_SUBSCRIPTION_ID/resourceGroups/YOUR_RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts/YOUR_ACCOUNT_NAME - -# Output: -# { -# "appId": "CLIENT_ID", -# "password": "CLIENT_SECRET", -# "tenant": "TENANT_ID" -# } -``` - -**Configuration in solr.xml:** +### Connection String (Development) ```xml - - - - YOUR_CONTAINER_NAME - https://YOUR_ACCOUNT_NAME.blob.core.windows.net - YOUR_TENANT_ID - YOUR_CLIENT_ID - YOUR_CLIENT_SECRET - - +DefaultEndpointsProtocol=https;AccountName=...;AccountKey=...;EndpointSuffix=core.windows.net ``` -**Alternative: Environment Variables** - -Instead of putting credentials in solr.xml, you can use environment variables: -```bash -export AZURE_TENANT_ID="your-tenant-id" -export AZURE_CLIENT_ID="your-client-id" -export AZURE_CLIENT_SECRET="your-client-secret" -``` +### SAS Token (Production) -Then solr.xml only needs: +Generate a SAS token with permissions: Read, Write, Delete, List, Add, Create (`sp=rwdlac`) and resource types: Service, Container, Object (`srt=sco`). ```xml - - - - YOUR_CONTAINER_NAME - https://YOUR_ACCOUNT_NAME.blob.core.windows.net - - - +https://YOUR_ACCOUNT.blob.core.windows.net +sv=2024-11-04&ss=b&srt=sco&sp=rwdlac&... ``` ---- +Note: Escape `&` as `&` in XML. -#### Option C: Managed Identity (Azure VM/AKS) +### Azure Identity (Production - Recommended) -Best for production workloads running on Azure infrastructure. **Most secure** - no credentials at all! - -**Enable Managed Identity:** -```bash -# For Azure VM -az vm identity assign \ - --name YOUR_VM_NAME \ - --resource-group YOUR_RESOURCE_GROUP - -# Get the managed identity principal ID -PRINCIPAL_ID=$(az vm identity show \ - --name YOUR_VM_NAME \ - --resource-group YOUR_RESOURCE_GROUP \ - --query principalId -o tsv) - -# Grant Storage Blob Data Contributor role -az role assignment create \ - --role "Storage Blob Data Contributor" \ - --assignee $PRINCIPAL_ID \ - --scope /subscriptions/YOUR_SUBSCRIPTION_ID/resourceGroups/YOUR_RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts/YOUR_ACCOUNT_NAME -``` - -**Configuration in solr.xml:** +Uses Azure AD authentication. Requires "Storage Blob Data Contributor" role on the storage account. ```xml - - - - YOUR_CONTAINER_NAME - https://YOUR_ACCOUNT_NAME.blob.core.windows.net - - - +https://YOUR_ACCOUNT.blob.core.windows.net + ``` ---- - -#### Why Use Azure Identity? - -**Security Benefits:** -- ✅ **Zero secrets** in configuration files -- ✅ **Automatic credential rotation** via Azure AD -- ✅ **Fine-grained RBAC** access control -- ✅ **Full audit logging** via Azure AD -- ✅ **Compliance-friendly** (SOC 2, ISO 27001, etc.) -- ✅ **Token-based** authentication (short-lived tokens) - -**Operational Benefits:** -- ✅ **No credential management** overhead -- ✅ **Works across environments** (dev, staging, prod) -- ✅ **Integrates with Azure services** seamlessly -- ✅ **Supports multiple identities** (users, service principals, managed identities) - -**Performance:** -- Slightly slower than key-based auth (~5-10 seconds overhead for token acquisition) -- Negligible for large backups (overhead is constant, not proportional to data size) -- Well worth the security benefits - -## Authentication Comparison - -| Method | Security | Setup | Best For | Credentials in Config | Production | -|--------|----------|-------|----------|----------------------|------------| -| Connection String | ⚠️ Low | ⭐ Simple | Development | ❌ Full account key | ❌ Dev only | -| Account Key | ⚠️ Low | ⭐ Simple | Development | ❌ Full account key | ⚠️ Caution | -| **SAS Token** | ✅ Good | ⭐⭐ Medium | **Production** | ⚠️ Time-limited token | ✅ **Recommended** | -| Azure Identity (CLI) | ✅ Excellent | ⭐⭐ Medium | Local Dev/Test | ✅ None (uses login) | ✅ Dev/Test | -| **Azure Identity (SP)** | ✅ Excellent | ⭐⭐⭐ Complex | **CI/CD/Production** | ⚠️ Scoped credentials | ✅ **Recommended** | -| Azure Identity (MI) | ✅✅ Best | ⭐⭐⭐ Complex | **Azure VMs/AKS** | ✅ **None** | ✅✅ **Best** | - -## Troubleshooting - -### SAS Token Issues - -**Error: "Failed to check existence" or "403 Forbidden"** - -This usually means the SAS token lacks required permissions. Verify: -1. ✅ Resource types include: **Service, Container, and Object** (`srt=sco`) - - ❌ Wrong: `srt=c` (container only) - - ✅ Correct: `srt=sco` (service, container, object) -2. ✅ Permissions include at least: **Read, Write, Delete, List, Add, Create** (`sp=rwdlac`) -3. ✅ Token has not expired -4. ✅ `&` characters are escaped as `&` in XML -5. ✅ Container already exists in Azure Blob Storage - -**Error: "Signature did not match"** - -1. Check that `&` characters are properly escaped as `&` in solr.xml -2. Ensure no extra whitespace or line breaks in the token -3. Remove the leading `?` from the token if present -4. Verify the token was copied completely - -### Azure Identity Issues - -**Error: "403 Forbidden" or "AuthorizationFailed"** - -This means your identity lacks the required permissions. Verify: - -1. ✅ **Azure CLI:** You're logged in with `az login` -2. ✅ **RBAC Role:** Identity has "Storage Blob Data Contributor" role -3. ✅ **Scope:** Role is assigned at the correct scope (storage account level) -4. ✅ **Token:** For CLI, run `az account get-access-token --resource https://storage.azure.com/` to verify token - -**Check role assignment:** -```bash -# List all role assignments for the storage account -az role assignment list \ - --scope /subscriptions/YOUR_SUBSCRIPTION_ID/resourceGroups/YOUR_RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts/YOUR_ACCOUNT_NAME \ - --query "[].{Principal:principalName, Role:roleDefinitionName}" -o table +For Service Principal, add: +```xml +YOUR_TENANT_ID +YOUR_CLIENT_ID +YOUR_CLIENT_SECRET ``` -**Error: "DefaultAzureCredential failed to retrieve token"** - -This means the credential chain couldn't find valid credentials. Check: - -1. **Azure CLI:** Ensure `az login` is successful and not expired -2. **Service Principal:** Verify environment variables or solr.xml credentials are correct -3. **Managed Identity:** Ensure it's enabled on the VM/AKS and has permissions -4. **Token expiry:** Azure CLI tokens expire - re-run `az login` if needed - -**Performance slower than expected:** - -Azure Identity adds ~5-10 seconds overhead for token acquisition. This is normal and expected: -- First operation: ~10-15 seconds (token acquisition) -- Subsequent operations: ~5 seconds (token refresh) -- For large backups (GB/TB), this overhead is negligible +Or set environment variables: `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`. ## Usage -Once you've configured authentication in `solr.xml`, you can use standard Solr backup/restore commands. - -### Create a Backup - ```bash -# Create a backup of a collection +# Backup curl "http://localhost:8983/solr/admin/collections?action=BACKUP&name=my-backup&collection=my-collection&repository=azure_blob&location=/" -# Example response: -# { -# "responseHeader": {"status": 0, "QTime": 1234}, -# "response": { -# "collection": "my-collection", -# "backupId": 1, -# "indexFileCount": 156, -# "indexSizeMB": 245.5 -# } -# } -``` - -**Parameters:** -- `name` - Backup name (used for restore) -- `collection` - Source collection to backup -- `repository` - Repository name from solr.xml (e.g., `blob`) -- `location` - Path in blob container (use `/` for root, or `/backups/` for subdirectory) - -### Restore from Backup +# Restore +curl "http://localhost:8983/solr/admin/collections?action=RESTORE&name=my-backup&collection=my-collection&repository=azure_blob&location=/" -```bash -# Restore a backup to a new or existing collection -curl "http://localhost:8983/solr/admin/collections?action=RESTORE&name=my-backup&collection=my-collection-restored&repository=azure_blob&location=/" - -# Example response: -# { -# "responseHeader": {"status": 0, "QTime": 567}, -# "success": {...} -# } -``` - -**Parameters:** -- `name` - Backup name to restore -- `collection` - Target collection name (can be different from original) -- `repository` - Repository name from solr.xml -- `location` - Same path used during backup - -### List Backups - -```bash -# List all backups at a location +# List backups curl "http://localhost:8983/solr/admin/collections?action=LISTBACKUP&name=my-backup&repository=azure_blob&location=/" - -# Example response: -# { -# "responseHeader": {"status": 0}, -# "backups": [ -# {"backupId": 1, "indexFileCount": 156, "indexSizeMB": 245.5}, -# {"backupId": 2, "indexFileCount": 158, "indexSizeMB": 247.1} -# ] -# } ``` -### Delete a Backup - -```bash -# Delete a specific backup -curl "http://localhost:8983/solr/admin/collections?action=DELETEBACKUP&name=my-backup&backupId=1&repository=azure_blob&location=/" -``` - -**Note:** The `location` parameter should be `/` (root of container) or a subdirectory path like `/backups/`. The path must not have a trailing slash except for root. - -### Best Practices - -1. **Naming Convention:** Use descriptive backup names with timestamps - ```bash - curl "...&name=my-collection-2025-10-08&..." - ``` +## Troubleshooting -2. **Regular Testing:** Periodically test restore operations - ```bash - # Restore to a test collection - curl "...&collection=my-collection-test&..." - ``` +**403 Forbidden**: Check SAS token permissions (`srt=sco`, `sp=rwdlac`) or RBAC role assignment. -3. **Multiple Backups:** Keep multiple backup versions - ```bash - # Backups are versioned automatically (backupId) - curl "...action=LISTBACKUP..." # View all versions - ``` +**Signature did not match**: Ensure `&` is escaped as `&` in XML and no whitespace in token. -4. **Monitor Progress:** Use Solr admin UI or check logs - ```bash - tail -f $SOLR_HOME/logs/solr.log | grep -i backup - ``` +**DefaultAzureCredential failed**: Run `az login` or verify service principal credentials. diff --git a/solr/modules/azure-blob-repository/build.gradle b/solr/modules/azure-blob-repository/build.gradle index ec9597362e31..df679db2c4b0 100644 --- a/solr/modules/azure-blob-repository/build.gradle +++ b/solr/modules/azure-blob-repository/build.gradle @@ -19,6 +19,12 @@ apply plugin: 'java-library' description = 'Azure Blob Storage Repository' +ext { + // Disable security manager for azure-blob-repository module tests + // Required because Testcontainers needs access to Docker socket and system properties + useSecurityManager = false +} + dependencies { implementation platform(project(':platform')) api(project(':solr:core')) @@ -54,6 +60,10 @@ dependencies { // OkHttp for test client management testImplementation libs.azure.core.http.okhttp + // Testcontainers for Azurite integration testing + testImplementation libs.testcontainers + // Explicit transitive test dependencies for dependency analyzer testImplementation 'com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.3' + testImplementation libs.apache.lucene.testframework } \ No newline at end of file diff --git a/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobBackupRepository.java b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobBackupRepository.java index 934798ee7d0f..50c8b63988af 100644 --- a/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobBackupRepository.java +++ b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobBackupRepository.java @@ -16,6 +16,7 @@ */ package org.apache.solr.azureblob; +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -50,6 +51,7 @@ public class AzureBlobBackupRepository extends AbstractBackupRepository { static final String BLOB_SCHEME = "blob"; private static final int CHUNK_SIZE = 16 * 1024 * 1024; + private static final int COPY_BUFFER_SIZE = 8192; private AzureBlobStorageClient client; @@ -58,7 +60,6 @@ public void init(NamedList args) { super.init(args); AzureBlobBackupRepositoryConfig backupConfig = new AzureBlobBackupRepositoryConfig(this.config); - // If a client was already created, close it to avoid any resource leak if (client != null) { client.close(); } @@ -66,7 +67,7 @@ public void init(NamedList args) { this.client = backupConfig.buildClient(); } - // Method to inject a mock client for testing + @VisibleForTesting public void setClient(AzureBlobStorageClient client) { this.client = client; } @@ -175,7 +176,7 @@ public void delete(URI path, Collection files) throws IOException { Objects.requireNonNull(files, "cannot delete with a null files collection"); String basePath = getBlobPath(path); - // If a file path was passed instead of a directory, use its parent directory as base + try { if (!client.isDirectory(basePath)) { int lastSlash = basePath.lastIndexOf('/'); @@ -309,7 +310,6 @@ public void copyIndexFileFrom( log.debug("Copy index file from '{}' to '{}'", sourceFileName, blobPath); } - // Ensure destination parent directory exists String parentDir = blobPath.contains("/") ? blobPath.substring(0, blobPath.lastIndexOf('/') + 1) : ""; try { @@ -317,13 +317,12 @@ public void copyIndexFileFrom( client.createDirectory(parentDir); } } catch (AzureBlobException e) { - // ignore failures here; write will surface real issues + // ignore; write will surface real issues } try (IndexInput input = sourceDir.openInput(sourceFileName, IOContext.DEFAULT); OutputStream output = client.pushStream(blobPath)) { - // Copy bytes from IndexInput to OutputStream - byte[] buffer = new byte[8192]; + byte[] buffer = new byte[COPY_BUFFER_SIZE]; long remaining = input.length(); while (remaining > 0) { int toRead = (int) Math.min(buffer.length, remaining); @@ -336,14 +335,6 @@ public void copyIndexFileFrom( } } - /** - * Copy an index file from specified sourceRepo to the destination directory (i.e. - * restore). - * - * @param sourceDir The source URI hosting the file to be copied. - * @param dest The destination where the file should be copied. - * @throws IOException in case of errors. - */ @Override public void copyIndexFileTo( URI sourceDir, String sourceFileName, Directory dest, String destFileName) @@ -357,7 +348,6 @@ public void copyIndexFileTo( String basePath = getBlobPath(sourceDir); String blobPath; - // If sourceDir already points to the file, avoid duplicating the name if (basePath.endsWith("/" + sourceFileName) || basePath.equals(sourceFileName) || basePath.equals("/" + sourceFileName)) { @@ -374,7 +364,6 @@ public void copyIndexFileTo( try (InputStream inputStream = client.pullStream(blobPath); IndexOutput indexOutput = dest.createOutput(destFileName, IOContext.DEFAULT)) { - // Copy bytes from InputStream to IndexOutput byte[] buffer = new byte[CHUNK_SIZE]; int len; while ((len = inputStream.read(buffer)) != -1) { diff --git a/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobBackupRepositoryConfig.java b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobBackupRepositoryConfig.java index ef960dfe9f22..f0f8f9c1f4c7 100644 --- a/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobBackupRepositoryConfig.java +++ b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobBackupRepositoryConfig.java @@ -19,7 +19,6 @@ import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.NamedList; -/** Class representing the {@code backup} Blob Storage config bundle specified in solr.xml. */ public class AzureBlobBackupRepositoryConfig { public static final String CONTAINER_NAME = "azure.blob.container.name"; @@ -54,7 +53,6 @@ public AzureBlobBackupRepositoryConfig(NamedList config) { clientSecret = getStringConfig(config, CLIENT_SECRET); } - /** Construct a {@link AzureBlobStorageClient} from the provided config. */ public AzureBlobStorageClient buildClient() { return new AzureBlobStorageClient( containerName, diff --git a/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobIndexInput.java b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobIndexInput.java index fc935b2a4c52..c523307e4f2e 100644 --- a/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobIndexInput.java +++ b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobIndexInput.java @@ -25,8 +25,9 @@ class AzureBlobIndexInput extends IndexInput { - private static final int DEFAULT_PAGE_SIZE = 512 * 1024; // 512 KB - private static final int MAX_CACHED_PAGES = 128; // ~64 MB at 512 KB pages + private static final int MIN_PAGE_SIZE = 4 * 1024; + private static final int DEFAULT_PAGE_SIZE = 512 * 1024; + private static final int MAX_CACHED_PAGES = 128; private final String path; private final AzureBlobStorageClient client; @@ -47,7 +48,7 @@ class AzureBlobIndexInput extends IndexInput { this.path = path; this.client = client; this.length = length; - this.pageSize = Math.max(4 * 1024, pageSize); + this.pageSize = Math.max(MIN_PAGE_SIZE, pageSize); this.cache = new LruPageCache(maxCachedPages); } diff --git a/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobOutputStream.java b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobOutputStream.java index 61388c983889..d48fc472a7e7 100644 --- a/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobOutputStream.java +++ b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobOutputStream.java @@ -39,7 +39,6 @@ public class AzureBlobOutputStream extends OutputStream { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - // 4 MB per block (Azure limit is 100 MB, but 4 MB is more efficient for most use cases) static final int BLOCK_SIZE = 4 * 1024 * 1024; private final BlobClient blobClient; @@ -70,7 +69,6 @@ public void write(int b) throws IOException { buffer.put((byte) b); - // If the buffer is now full, push it to Azure Blob Storage if (!buffer.hasRemaining()) { uploadBlock(); } @@ -111,7 +109,6 @@ private void uploadBlock() throws IOException { int size = buffer.position() - buffer.arrayOffset(); if (size == 0) { - // nothing to upload return; } @@ -119,6 +116,7 @@ private void uploadBlock() throws IOException { if (log.isDebugEnabled()) { log.debug("New block upload for blobPath '{}'", blobPath); } + blockUpload = newBlockUpload(); } @@ -132,11 +130,11 @@ private void uploadBlock() throws IOException { log.debug("Block upload aborted for blobPath '{}'.", blobPath); } } + throw new IOException( "Failed to upload block", AzureBlobStorageClient.handleBlobException(e)); } - // reset the buffer for eventual next write operation buffer.clear(); } @@ -146,12 +144,10 @@ public void flush() throws IOException { throw new IOException("Stream closed"); } - // Ensure any buffered data is staged to Azure if (buffer.position() - buffer.arrayOffset() > 0) { uploadBlock(); } - // Make data visible by committing current block list (idempotent, can be called again on close) if (blockUpload != null) { blockUpload.complete(); blockUpload = null; @@ -172,14 +168,12 @@ public void close() throws IOException { } if (!committed) { - // Stage any remaining data and commit once uploadBlock(); if (blockUpload != null) { blockUpload.complete(); blockUpload = null; committed = true; } else { - // No data was written; ensure a zero-length blob exists at this path try { blobClient.upload(new ByteArrayInputStream(new byte[0]), 0, true); } catch (BlobStorageException e) { @@ -188,13 +182,12 @@ public void close() throws IOException { } } } else { - // Already committed via flush. If additional writes occurred after flush, - // there will be a new blockUpload. Commit it to overwrite previous content. if (blockUpload != null) { blockUpload.complete(); blockUpload = null; } } + closed = true; } @@ -216,13 +209,12 @@ public BlockUpload() { if (log.isDebugEnabled()) { log.debug("Initiated block upload for blobPath '{}'", blobPath); } - // Ensure we start with a clean slate; if a blob already exists at this path, - // remove it so that the commit does not fail with BlobAlreadyExists (409). + try { BlockBlobClient blockBlobClient = blobClient.getBlockBlobClient(); blockBlobClient.deleteIfExists(); } catch (BlobStorageException e) { - // Ignore deletion problems here; subsequent stage/commit will surface real issues + // ignore; subsequent stage/commit will surface real issues } } @@ -249,7 +241,6 @@ void uploadBlock(ByteArrayInputStream inputStream, long blockSize) { } } - /** To be invoked when closing the stream to mark upload is done. */ void complete() { if (aborted) { throw new IllegalStateException("Can't complete a BlockUpload that was aborted"); @@ -272,9 +263,6 @@ public void abort() { log.warn("Aborting block upload for blobPath '{}'", blobPath); } - // Azure doesn't have an explicit abort operation for block uploads - // The blocks will remain as uncommitted blocks and will be cleaned up - // by Azure's garbage collection after 7 days aborted = true; } } diff --git a/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobStorageClient.java b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobStorageClient.java index 217ff2975781..e91b8d6dcbbb 100644 --- a/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobStorageClient.java +++ b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobStorageClient.java @@ -50,12 +50,11 @@ public class AzureBlobStorageClient { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); static final String BLOB_FILE_PATH_DELIMITER = "/"; + private static final int HTTP_NOT_FOUND = 404; + private static final int HTTP_CONFLICT = 409; + private static final int SKIP_BUFFER_SIZE = 8192; + private static final int DELETE_BATCH_SIZE = 1000; - /** - * Shared HttpClient instance for all Azure Blob Storage operations. OkHttp recommends reusing a - * single OkHttpClient instance as it maintains connection pools and thread pools that are - * expensive to create. This also prevents thread leaks in tests by using shared global threads. - */ private static final com.azure.core.http.HttpClient SHARED_HTTP_CLIENT = new com.azure.core.http.okhttp.OkHttpAsyncHttpClientBuilder().build(); @@ -90,7 +89,7 @@ public class AzureBlobStorageClient { try { containerClient.create(); } catch (BlobStorageException e) { - if (e.getStatusCode() != 409) { + if (e.getStatusCode() != HTTP_CONFLICT) { throw e; } } @@ -107,7 +106,6 @@ private static BlobServiceClient createInternalClient( String clientSecret) { BlobServiceClientBuilder builder = new BlobServiceClientBuilder(); - // Use shared OkHttp client for better resource management builder.httpClient(SHARED_HTTP_CLIENT); if (StrUtils.isNotNullOrEmpty(connectionString)) { @@ -120,7 +118,6 @@ private static BlobServiceClient createInternalClient( } else if (StrUtils.isNotNullOrEmpty(sasToken)) { builder.sasToken(sasToken); } else { - // Use default Azure credential provider chain TokenCredential credential = new DefaultAzureCredentialBuilder().tenantId(tenantId).build(); builder.credential(credential); } @@ -131,20 +128,16 @@ private static BlobServiceClient createInternalClient( return builder.buildClient(); } - /** Create a directory in Blob Storage, if it does not already exist. */ void createDirectory(String path) throws AzureBlobException { String sanitizedDirPath = sanitizedDirPath(path); - // Only create the directory if it does not already exist if (!pathExists(sanitizedDirPath)) { String parent = getParentDirectory(sanitizedDirPath); - // Stop at root if (!parent.isEmpty() && !parent.equals(BLOB_FILE_PATH_DELIMITER)) { createDirectory(parent); } try { - // Create empty blob and mark it as a directory via metadata BlobClient blobClient = containerClient.getBlobClient(sanitizedDirPath); blobClient.upload(new ByteArrayInputStream(new byte[0]), 0, true); java.util.Map metadata = new java.util.HashMap<>(); @@ -156,7 +149,6 @@ void createDirectory(String path) throws AzureBlobException { } } - /** Delete files from Blob Storage. Missing files are ignored (idempotent delete). */ void delete(Collection paths) throws AzureBlobException { Set entries = new HashSet<>(); for (String path : paths) { @@ -165,11 +157,9 @@ void delete(Collection paths) throws AzureBlobException { deleteBlobs(entries); } - /** Delete directory, all the files and subdirectories from Blob Storage. */ void deleteDirectory(String path) throws AzureBlobException { path = sanitizedDirPath(path); - // Get all the files and subdirectories Set entries = listAll(path); if (pathExists(path)) { entries.add(path); @@ -178,14 +168,13 @@ void deleteDirectory(String path) throws AzureBlobException { deleteBlobs(entries); } - /** List all the files and subdirectories directly under given path. */ String[] listDir(String path) throws AzureBlobException { path = sanitizedDirPath(path); try { ListBlobsOptions options = new ListBlobsOptions().setPrefix(path).setMaxResultsPerPage(1000); - final String finalPath = path; // Make path effectively final for lambda + final String finalPath = path; return containerClient.listBlobs(options, null).stream() .map(BlobItem::getName) .filter(s -> s.startsWith(finalPath)) @@ -202,11 +191,9 @@ String[] listDir(String path) throws AzureBlobException { } } - /** Check if path exists. */ boolean pathExists(String path) throws AzureBlobException { final String blobPath = sanitizedPath(path); - // for root return true if (blobPath.isEmpty() || BLOB_FILE_PATH_DELIMITER.equals(blobPath)) { return true; } @@ -219,27 +206,22 @@ boolean pathExists(String path) throws AzureBlobException { } } - /** Check if path is directory. */ boolean isDirectory(String path) throws AzureBlobException { final String dirPrefix = sanitizedDirPath(path); try { - // First, if there are any child blobs under this prefix, it's a directory ListBlobsOptions options = new ListBlobsOptions().setPrefix(dirPrefix).setMaxResultsPerPage(1); if (containerClient.listBlobs(options, null).iterator().hasNext()) { return true; } - // Otherwise, check if an empty blob exactly named with the trailing slash exists BlobClient markerClient = containerClient.getBlobClient(dirPrefix); if (markerClient.exists()) { long size = markerClient.getProperties().getBlobSize(); if (size == 0) { - // zero-byte marker with name ending in '/' is a directory return true; } - // If it's a non-zero blob at a name with '/', treat conservatively as file java.util.Map md = markerClient.getProperties().getMetadata(); return md != null && md.containsKey("hdi_isfolder"); } @@ -250,7 +232,6 @@ boolean isDirectory(String path) throws AzureBlobException { } } - /** Get length of file in bytes. */ long length(String path) throws AzureBlobException { String blobPath = sanitizedFilePath(path); try { @@ -261,7 +242,6 @@ long length(String path) throws AzureBlobException { } } - /** Open a new {@link InputStream} to file for read. */ InputStream pullStream(String path) throws AzureBlobException { final String blobPath = sanitizedFilePath(path); @@ -269,6 +249,10 @@ InputStream pullStream(String path) throws AzureBlobException { BlobClient blobClient = containerClient.getBlobClient(blobPath); final long contentLength = blobClient.getProperties().getBlobSize(); + if (contentLength == 0) { + return new ByteArrayInputStream(new byte[0]); + } + InputStream initial = new IdempotentCloseInputStream(blobClient.openInputStream()); return new ResumableInputStream( @@ -282,7 +266,6 @@ InputStream pullStream(String path) throws AzureBlobException { contentLength > 0 ? Math.max(0, contentLength - bytesRead) : Long.MAX_VALUE; return pullRangeStream(path, bytesRead, remaining); } catch (AzureBlobException e) { - // ResumableInputStream supplier cannot throw checked exceptions throw new RuntimeException(e); } }); @@ -291,7 +274,6 @@ InputStream pullStream(String path) throws AzureBlobException { } } - /** Open a ranged {@link InputStream} to file for read from offset for length bytes. */ InputStream pullRangeStream(String path, long offset, long length) throws AzureBlobException { final String blobPath = sanitizedFilePath(path); try { @@ -304,7 +286,6 @@ InputStream pullRangeStream(String path, long offset, long length) throws AzureB } } - /** Wrapper that makes close() idempotent (second close is a no-op). */ private static final class IdempotentCloseInputStream extends FilterInputStream { private boolean closed; @@ -370,7 +351,7 @@ public long skip(long n) throws java.io.IOException { return 0L; } long remaining = n; - byte[] discard = new byte[8192]; + byte[] discard = new byte[SKIP_BUFFER_SIZE]; try { while (remaining > 0) { int toRead = (int) Math.min(discard.length, remaining); @@ -382,7 +363,6 @@ public long skip(long n) throws java.io.IOException { } return n - remaining; } catch (RuntimeException re) { - // Normalize runtime issues from Azure's stream into IOExceptions so upper layers can resume throw new java.io.IOException(re); } } @@ -393,12 +373,10 @@ private static boolean isAlreadyClosed(Throwable t) { } } - /** Open a new {@link OutputStream} to file for write. */ OutputStream pushStream(String path) throws AzureBlobException { path = sanitizedFilePath(path); if (!parentDirectoryExist(path)) { - // Auto-create missing parent directory to mirror Azure's virtual directory semantics String parentDirectory = getParentDirectory(path); if (!parentDirectory.isEmpty() && !parentDirectory.equals(BLOB_FILE_PATH_DELIMITER)) { createDirectory(parentDirectory); @@ -413,18 +391,14 @@ OutputStream pushStream(String path) throws AzureBlobException { } } - /** Close the client. */ - void close() { - // Azure SDK clients don't need explicit closing - } + void close() {} @VisibleForTesting void deleteContainerForTests() { try { containerClient.delete(); } catch (BlobStorageException e) { - // Ignore not found - if (e.getStatusCode() != 404) { + if (e.getStatusCode() != HTTP_NOT_FOUND) { throw e; } } @@ -432,7 +406,7 @@ void deleteContainerForTests() { private Collection deleteBlobs(Collection paths) throws AzureBlobException { try { - return deleteBlobs(paths, 1000); // Azure supports batch delete + return deleteBlobs(paths, DELETE_BATCH_SIZE); } catch (BlobStorageException e) { throw handleBlobException(e); } @@ -451,10 +425,10 @@ Collection deleteBlobs(Collection entries, int batchSize) deletedPaths.add(path); } } catch (BlobStorageException e) { - if (e.getStatusCode() == 404) { - // ignore missing + if (e.getStatusCode() == HTTP_NOT_FOUND) { continue; } + throw new AzureBlobException("Could not delete blob with path: " + path, e); } } @@ -502,17 +476,15 @@ private String getParentDirectory(String path) { : ""; } - /** Ensures path adheres to some rules: -Doesn't start with a leading slash */ String sanitizedPath(String path) throws AzureBlobException { String sanitizedPath = path.trim(); - // Remove all leading slashes so that blob names never start with '/' while (sanitizedPath.startsWith(BLOB_FILE_PATH_DELIMITER)) { sanitizedPath = sanitizedPath.substring(1).trim(); } + return sanitizedPath; } - /** Ensures file path adheres to some rules */ String sanitizedFilePath(String path) throws AzureBlobException { String sanitizedPath = sanitizedPath(path); @@ -527,7 +499,6 @@ String sanitizedFilePath(String path) throws AzureBlobException { return sanitizedPath; } - /** Ensures directory path adheres to some rules */ String sanitizedDirPath(String path) throws AzureBlobException { String sanitizedPath = sanitizedPath(path); @@ -538,7 +509,6 @@ String sanitizedDirPath(String path) throws AzureBlobException { return sanitizedPath; } - /** Handle Azure Blob Storage exceptions */ static AzureBlobException handleBlobException(BlobStorageException e) { String errMessage = String.format( @@ -550,7 +520,7 @@ static AzureBlobException handleBlobException(BlobStorageException e) { log.error(errMessage); - if (e.getStatusCode() == 404) { + if (e.getStatusCode() == HTTP_NOT_FOUND) { return new AzureBlobNotFoundException(errMessage, e); } else { return new AzureBlobException(errMessage, e); diff --git a/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/package-info.java b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/package-info.java index 8be0e21aca24..c76136b3e788 100644 --- a/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/package-info.java +++ b/solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/package-info.java @@ -15,33 +15,5 @@ * limitations under the License. */ -/** - * Azure Blob Storage backup repository implementation for Apache Solr. - * - *

This package provides a {@link org.apache.solr.azureblob.AzureBlobBackupRepository} - * implementation that enables Solr to store and retrieve backup data from Azure Blob Storage. - * - *

The repository supports various Azure authentication methods including: - * - *

    - *
  • Connection strings - *
  • Account name and key - *
  • SAS tokens - *
  • Azure Identity (Managed Identity, Service Principal) - *
- * - *

Key components: - * - *

    - *
  • {@link org.apache.solr.azureblob.AzureBlobBackupRepository} - Main repository - * implementation - *
  • {@link org.apache.solr.azureblob.AzureBlobStorageClient} - Azure Blob Storage client - * wrapper - *
  • {@link org.apache.solr.azureblob.AzureBlobBackupRepositoryConfig} - Configuration - * management - *
- * - * @see Azure Blob Storage - * Documentation - */ +/** Solr Azure Blob Storage backup repository */ package org.apache.solr.azureblob; diff --git a/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AbstractAzureBlobClientTest.java b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AbstractAzureBlobClientTest.java index 3aee901d2aa2..9aaf731466f3 100644 --- a/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AbstractAzureBlobClientTest.java +++ b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AbstractAzureBlobClientTest.java @@ -17,62 +17,82 @@ package org.apache.solr.azureblob; import com.azure.core.http.HttpClient; -import com.azure.core.http.netty.NettyAsyncHttpClientBuilder; +import com.azure.core.http.okhttp.OkHttpAsyncHttpClientBuilder; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; +import com.carrotsearch.randomizedtesting.ThreadFilter; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.concurrent.TimeUnit; +import okhttp3.OkHttpClient; +import org.apache.lucene.tests.util.QuickPatchThreadsFilter; +import org.apache.solr.SolrIgnoredThreadsFilter; import org.apache.solr.SolrTestCaseJ4; import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assume; import org.junit.Before; -import org.junit.Rule; -import org.junit.rules.TemporaryFolder; -import reactor.netty.resources.ConnectionProvider; +import org.junit.BeforeClass; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; /** Abstract class for tests with Azure Blob Storage emulator. */ +@ThreadLeakFilters( + defaultFilters = true, + filters = { + SolrIgnoredThreadsFilter.class, + QuickPatchThreadsFilter.class, + AbstractAzureBlobClientTest.OkHttpThreadLeakFilterTest.class, + }) public class AbstractAzureBlobClientTest extends SolrTestCaseJ4 { - protected String containerName; - - @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); + private static final String AZURITE_IMAGE = "mcr.microsoft.com/azure-storage/azurite:3.33.0"; + private static final int BLOB_SERVICE_PORT = 10000; - AzureBlobStorageClient client; + private static GenericContainer azuriteContainer; + private static OkHttpClient sharedOkHttpClient; private static String connectionString; - private EventLoopGroup eventLoopGroup; - private ConnectionProvider connectionProvider; + + protected String containerName; protected org.apache.solr.client.solrj.cloud.SocketProxy proxy; + protected AzureBlobStorageClient client; + + @SuppressWarnings("resource") + @BeforeClass + public static void setUpClass() { + try { + azuriteContainer = + new GenericContainer<>(DockerImageName.parse(AZURITE_IMAGE)) + .withExposedPorts(BLOB_SERVICE_PORT); + azuriteContainer.start(); + sharedOkHttpClient = new OkHttpClient.Builder().build(); + } catch (Throwable t) { + Assume.assumeNoException("Docker/Testcontainers not available; skipping Azure tests", t); + } + } + @Before public void setUpClient() throws Exception { setAzureTestCredentials(); - // Disable Netty Flight Recorder to avoid Security Manager issues - // Keep default Netty client; OkHttp dependency not present - - // Use Azurite connection string for local testing + String blobServiceUrl = getBlobServiceUrl(); connectionString = - "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;"; - - // Build a Netty HTTP client with isolated resources we can shut down after tests - connectionProvider = ConnectionProvider.create("solr-azure-test"); - eventLoopGroup = new NioEventLoopGroup(1); + "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=" + + blobServiceUrl + + "/devstoreaccount1;"; - // Put a proxy in front of Azurite to simulate connection loss like S3 tests proxy = new org.apache.solr.client.solrj.cloud.SocketProxy(); - proxy.open(new java.net.URI(getBlobServiceUrl())); + proxy.open(new java.net.URI(blobServiceUrl)); - HttpClient httpClient = - new NettyAsyncHttpClientBuilder() - .connectionProvider(connectionProvider) - .eventLoopGroup(eventLoopGroup) - .build(); + HttpClient httpClient = new OkHttpAsyncHttpClientBuilder(sharedOkHttpClient).build(); + + String proxiedConn = + connectionString.replace( + ":" + azuriteContainer.getMappedPort(BLOB_SERVICE_PORT), ":" + proxy.getListenPort()); - // Route Blob endpoint through the proxy by adjusting the connection string - String proxiedConn = connectionString.replace(":10000", ":" + proxy.getListenPort()); BlobServiceClient blobServiceClient = new BlobServiceClientBuilder() .connectionString(proxiedConn) @@ -83,20 +103,10 @@ public void setUpClient() throws Exception { client = new AzureBlobStorageClient(blobServiceClient, containerName); } - /** - * Set up Azure test credentials to avoid using real Azure credentials during testing. Similar to - * how S3 tests use ProfileFileSystemSetting to avoid polluting the test environment. - */ public static void setAzureTestCredentials() { - // Set test Azure credentials to avoid using real credentials System.setProperty("AZURE_CLIENT_ID", "test-client-id"); System.setProperty("AZURE_TENANT_ID", "test-tenant-id"); System.setProperty("AZURE_CLIENT_SECRET", "test-client-secret"); - - // Set Azurite-specific environment variables - System.setProperty( - "AZURE_STORAGE_CONNECTION_STRING", - "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;"); } @After @@ -112,49 +122,50 @@ public void tearDownClient() { proxy.close(); proxy = null; } - try { - reactor.core.scheduler.Schedulers.shutdownNow(); - reactor.core.scheduler.Schedulers.resetFactory(); - } catch (Throwable ignored) { - } - - // Dispose custom Netty resources to prevent leaked threads - try { - if (connectionProvider != null) { - connectionProvider.disposeLater().block(); - } - } catch (Throwable ignored) { - } - try { - if (eventLoopGroup != null) { - eventLoopGroup.shutdownGracefully(0, 2, TimeUnit.SECONDS).awaitUninterruptibly(3000); - } - } catch (Throwable ignored) { - } } - /** Simulate a connection loss on the proxy similar to S3 tests. */ - void initiateBlobConnectionLoss() throws AzureBlobException { + /** Simulate a connection loss on the proxy. */ + void initiateBlobConnectionLoss() { if (proxy != null) { proxy.halfClose(); } } - @org.junit.AfterClass + @AfterClass public static void afterAll() { + if (azuriteContainer != null) { + try { + azuriteContainer.stop(); + azuriteContainer.close(); + } catch (Throwable ignored) { + } + azuriteContainer = null; + } + + if (sharedOkHttpClient != null) { + sharedOkHttpClient.dispatcher().executorService().shutdown(); + sharedOkHttpClient.dispatcher().cancelAll(); + sharedOkHttpClient.connectionPool().evictAll(); + try { + if (sharedOkHttpClient.cache() != null) { + sharedOkHttpClient.cache().close(); + } + } catch (Throwable ignored) { + } + try { + sharedOkHttpClient.dispatcher().executorService().awaitTermination(2, TimeUnit.SECONDS); + } catch (Throwable ignored) { + } + sharedOkHttpClient = null; + } + try { reactor.core.scheduler.Schedulers.shutdownNow(); - reactor.core.scheduler.Schedulers.resetFactory(); + Thread.sleep(100); } catch (Throwable ignored) { } } - /** - * Helper method to push a string to Azure Blob Storage. - * - * @param path Destination path in blob storage. - * @param content Arbitrary content for the test. - */ void pushContent(String path, String content) throws AzureBlobException { pushContent(path, content.getBytes(StandardCharsets.UTF_8)); } @@ -167,13 +178,26 @@ void pushContent(String path, byte[] content) throws AzureBlobException { } } - /** Get the connection string for tests that need direct access to the blob service. */ static String getConnectionString() { return connectionString; } - /** Get the blob service URL for tests that need direct access. */ String getBlobServiceUrl() { - return "http://localhost:10000"; + return "http://" + + azuriteContainer.getHost() + + ":" + + azuriteContainer.getMappedPort(BLOB_SERVICE_PORT); + } + + public static class OkHttpThreadLeakFilterTest implements ThreadFilter { + + @Override + public boolean reject(Thread t) { + String name = t.getName(); + if (name == null) { + return false; + } + return name.contains("OkHttp") || name.contains("Okio Watchdog"); + } } } diff --git a/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobBackupRepositoryTest.java b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobBackupRepositoryTest.java index 530b25c3a4c6..cc2432eb51c8 100644 --- a/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobBackupRepositoryTest.java +++ b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobBackupRepositoryTest.java @@ -51,26 +51,24 @@ protected URI getBaseUri() { return URI.create(BLOB_SCHEME + ":/"); } + @Override @Before public void setUp() throws Exception { super.setUp(); NamedList config = new NamedList<>(); - config.add("blob.container.name", CONTAINER_NAME); - config.add("blob.connection.string", getConnectionString()); + config.add("azure.blob.container.name", CONTAINER_NAME); + config.add("azure.blob.connection.string", getConnectionString()); - // Use a repository that avoids creating its own Azure client (which leaks Netty threads) - // and instead inject the pre-configured client from AbstractBlobClientTest. repository = new AzureBlobBackupRepository() { @Override public void init(NamedList args) { - // Only capture config; avoid building a new client inside init this.config = args; - // Inject the already-initialized client that uses isolated Netty resources setClient(AzureBlobBackupRepositoryTest.this.client); } }; + repository.init(config); } @@ -104,12 +102,10 @@ public void testReadWriteFile() throws IOException { URI fileUri = getBaseUri().resolve("read-write-test.txt"); String originalContent = "Test content for read/write operations"; - // Write content try (OutputStream output = repository.createOutput(fileUri)) { output.write(originalContent.getBytes(StandardCharsets.UTF_8)); } - // Read content try (IndexInput input = repository.openInput(getBaseUri(), "read-write-test.txt", IOContext.DEFAULT)) { byte[] buffer = new byte[1024]; @@ -124,14 +120,12 @@ public void testDeleteFile() throws IOException { URI fileUri = getBaseUri().resolve("delete-test.txt"); String content = "File to be deleted"; - // Create file try (OutputStream output = repository.createOutput(fileUri)) { output.write(content.getBytes(StandardCharsets.UTF_8)); } assertTrue("File should exist before deletion", repository.exists(fileUri)); - // Delete file repository.delete(fileUri, java.util.Arrays.asList("delete-test.txt")); assertFalse("File should not exist after deletion", repository.exists(fileUri)); @@ -142,7 +136,6 @@ public void testDeleteDirectory() throws IOException { URI dirUri = getBaseUri().resolve("delete-dir/"); URI fileUri = dirUri.resolve("nested-file.txt"); - // Create directory and file repository.createDirectory(dirUri); try (OutputStream output = repository.createOutput(fileUri)) { output.write("Nested file content".getBytes(StandardCharsets.UTF_8)); @@ -151,7 +144,6 @@ public void testDeleteDirectory() throws IOException { assertTrue("Directory should exist", repository.exists(dirUri)); assertTrue("File should exist", repository.exists(fileUri)); - // Delete directory repository.deleteDirectory(dirUri); assertFalse("Directory should not exist after deletion", repository.exists(dirUri)); @@ -163,7 +155,6 @@ public void testListDirectory() throws IOException { URI dirUri = getBaseUri().resolve("list-test/"); repository.createDirectory(dirUri); - // Create some files String[] fileNames = {"file1.txt", "file2.txt", "subdir/"}; for (String fileName : fileNames) { URI fileUri = dirUri.resolve(fileName); @@ -193,7 +184,6 @@ public void testListDirectory() throws IOException { @Test public void testCopyFileFromDirectory() throws IOException { - // Create a temporary directory with a file Path tempDir = Files.createTempDirectory("blob-test"); Path tempFile = tempDir.resolve("source-file.txt"); String content = "Source file content"; @@ -224,7 +214,6 @@ public void testCopyFileFromDirectory() throws IOException { @Test public void testCopyFileToDirectory() throws IOException { - // Create a file in blob storage URI sourceUri = getBaseUri().resolve("source-file.txt"); String content = "Source file content"; @@ -232,7 +221,6 @@ public void testCopyFileToDirectory() throws IOException { output.write(content.getBytes(StandardCharsets.UTF_8)); } - // Create a temporary directory Path tempDir = Files.createTempDirectory("blob-test"); try { @@ -257,12 +245,10 @@ public void testIndexInputOutput() throws IOException { URI fileUri = getBaseUri().resolve("index-test.txt"); String content = "Test content for index input/output"; - // Write using IndexOutput try (OutputStream output = repository.createOutput(fileUri)) { output.write(content.getBytes(StandardCharsets.UTF_8)); } - // Read using IndexInput try (IndexInput input = repository.openInput(getBaseUri(), "index-test.txt", IOContext.DEFAULT)) { byte[] buffer = new byte[(int) input.length()]; @@ -274,17 +260,14 @@ public void testIndexInputOutput() throws IOException { @Test public void testChecksumVerification() throws IOException { - // Create a file with checksum URI fileUri = getBaseUri().resolve("checksum-test.txt"); String content = "Test content for checksum verification"; try (OutputStream output = repository.createOutput(fileUri)) { output.write(content.getBytes(StandardCharsets.UTF_8)); - // Write a simple footer for testing output.write("FOOTER".getBytes(StandardCharsets.UTF_8)); } - // Verify content (skip checksum verification for this simple test) try (IndexInput input = repository.openInput(getBaseUri(), "checksum-test.txt", IOContext.DEFAULT)) { byte[] buffer = new byte[1024]; @@ -294,17 +277,10 @@ public void testChecksumVerification() throws IOException { } } - /** - * Provide a base {@link BackupRepository} configuration for use by any tests that call {@link - * BackupRepository#init(NamedList)} explicitly. - * - *

Useful for setting configuration properties required for specific BackupRepository - * implementations. - */ protected NamedList getBaseBackupRepositoryConfiguration() { NamedList config = new NamedList<>(); - config.add("blob.container.name", CONTAINER_NAME); - config.add("blob.connection.string", getConnectionString()); + config.add("azure.blob.container.name", CONTAINER_NAME); + config.add("azure.blob.connection.string", getConnectionString()); return config; } diff --git a/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobIncrementalBackupTest.java b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobIncrementalBackupTest.java index 68057dc6f8c8..417c80dc139c 100644 --- a/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobIncrementalBackupTest.java +++ b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobIncrementalBackupTest.java @@ -25,13 +25,9 @@ public class AzureBlobIncrementalBackupTest extends AbstractAzureBlobClientTest public void testIncrementalBackup() throws Exception { String backupPath = "incremental-backup-test/"; - // Create initial backup createBackup(backupPath + "backup1/", "Initial backup content"); - - // Create incremental backup createBackup(backupPath + "backup2/", "Incremental backup content"); - // Verify both backups exist assertTrue("Initial backup should exist", client.pathExists(backupPath + "backup1/")); assertTrue("Incremental backup should exist", client.pathExists(backupPath + "backup2/")); } @@ -39,8 +35,6 @@ public void testIncrementalBackup() throws Exception { @Test public void testBackupWithMultipleFiles() throws Exception { String backupPath = "multi-file-backup-test/"; - - // Create backup with multiple files String[] files = {"file1.txt", "file2.txt", "file3.txt"}; String[] contents = {"Content 1", "Content 2", "Content 3"}; @@ -48,7 +42,6 @@ public void testBackupWithMultipleFiles() throws Exception { pushContent(backupPath + files[i], contents[i]); } - // Verify all files exist for (String file : files) { assertTrue("File should exist: " + file, client.pathExists(backupPath + file)); } @@ -57,8 +50,6 @@ public void testBackupWithMultipleFiles() throws Exception { @Test public void testBackupWithNestedDirectories() throws Exception { String backupPath = "nested-backup-test/"; - - // Create nested directory structure String[] dirs = { backupPath + "level1/", backupPath + "level1/level2/", backupPath + "level1/level2/level3/" }; @@ -67,12 +58,10 @@ public void testBackupWithNestedDirectories() throws Exception { client.createDirectory(dir); } - // Add files at different levels pushContent(backupPath + "root-file.txt", "Root file content"); pushContent(backupPath + "level1/mid-file.txt", "Mid file content"); pushContent(backupPath + "level1/level2/level3/deep-file.txt", "Deep file content"); - // Verify structure assertTrue("Root file should exist", client.pathExists(backupPath + "root-file.txt")); assertTrue("Mid file should exist", client.pathExists(backupPath + "level1/mid-file.txt")); assertTrue( @@ -84,15 +73,12 @@ public void testBackupWithNestedDirectories() throws Exception { public void testBackupRestore() throws Exception { String backupPath = "backup-restore-test/"; String restorePath = "restore-test/"; - - // Create backup String originalContent = "Original backup content"; + pushContent(backupPath + "backup-file.txt", originalContent); - // Simulate restore by copying content try (var input = client.pullStream(backupPath + "backup-file.txt"); var output = client.pushStream(restorePath + "restored-file.txt")) { - byte[] buffer = new byte[1024]; int bytesRead; while ((bytesRead = input.read(buffer)) != -1) { @@ -100,10 +86,8 @@ public void testBackupRestore() throws Exception { } } - // Verify restore assertTrue("Restored file should exist", client.pathExists(restorePath + "restored-file.txt")); - // Verify content try (var input = client.pullStream(restorePath + "restored-file.txt")) { byte[] buffer = new byte[1024]; int bytesRead = input.read(buffer); @@ -115,8 +99,6 @@ public void testBackupRestore() throws Exception { @Test public void testBackupWithLargeFiles() throws Exception { String backupPath = "large-file-backup-test/"; - - // Create large file StringBuilder contentBuilder = new StringBuilder(); for (int i = 0; i < 10000; i++) { contentBuilder.append("This is line ").append(i).append(" of the large backup file.\n"); @@ -125,7 +107,6 @@ public void testBackupWithLargeFiles() throws Exception { pushContent(backupPath + "large-backup.txt", largeContent); - // Verify large file assertTrue( "Large backup file should exist", client.pathExists(backupPath + "large-backup.txt")); assertEquals( @@ -137,8 +118,6 @@ public void testBackupWithLargeFiles() throws Exception { @Test public void testBackupWithBinaryFiles() throws Exception { String backupPath = "binary-backup-test/"; - - // Create binary file byte[] binaryData = new byte[1024]; for (int i = 0; i < binaryData.length; i++) { binaryData[i] = (byte) (i % 256); @@ -146,7 +125,6 @@ public void testBackupWithBinaryFiles() throws Exception { pushContent(backupPath + "binary-backup.bin", binaryData); - // Verify binary file assertTrue( "Binary backup file should exist", client.pathExists(backupPath + "binary-backup.bin")); assertEquals( @@ -159,23 +137,19 @@ public void testBackupWithBinaryFiles() throws Exception { public void testBackupCleanup() throws Exception { String backupPath = "backup-cleanup-test/"; - // Create multiple backups for (int i = 1; i <= 5; i++) { pushContent(backupPath + "backup" + i + "/backup-file.txt", "Backup " + i + " content"); } - // Verify all backups exist for (int i = 1; i <= 5; i++) { assertTrue( "Backup " + i + " should exist", client.pathExists(backupPath + "backup" + i + "/")); } - // Cleanup old backups (keep only last 3) for (int i = 1; i <= 2; i++) { client.deleteDirectory(backupPath + "backup" + i + "/"); } - // Verify cleanup for (int i = 1; i <= 2; i++) { assertFalse( "Old backup " + i + " should not exist", @@ -192,13 +166,11 @@ public void testBackupCleanup() throws Exception { public void testBackupWithMetadata() throws Exception { String backupPath = "metadata-backup-test/"; - // Create backup with metadata files pushContent( backupPath + "backup-metadata.json", "{\"timestamp\":\"2023-01-01T00:00:00Z\",\"version\":\"1.0\"}"); pushContent(backupPath + "backup-data.txt", "Backup data content"); - // Verify metadata files assertTrue( "Metadata file should exist", client.pathExists(backupPath + "backup-metadata.json")); assertTrue("Data file should exist", client.pathExists(backupPath + "backup-data.txt")); @@ -207,17 +179,13 @@ public void testBackupWithMetadata() throws Exception { @Test public void testConcurrentBackups() throws Exception { String backupPath = "concurrent-backup-test/"; - - // Simulate concurrent backups String[] backupNames = {"backup1", "backup2", "backup3"}; String[] contents = {"Content 1", "Content 2", "Content 3"}; - // Create backups concurrently (simulated) for (int i = 0; i < backupNames.length; i++) { pushContent(backupPath + backupNames[i] + "/backup-file.txt", contents[i]); } - // Verify all backups exist for (String backupName : backupNames) { assertTrue( "Backup should exist: " + backupName, client.pathExists(backupPath + backupName + "/")); diff --git a/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobIndexInputTest.java b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobIndexInputTest.java index 7db9e286e93b..b91274fceea3 100644 --- a/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobIndexInputTest.java +++ b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobIndexInputTest.java @@ -27,10 +27,8 @@ public void testBasicIndexInput() throws Exception { String path = "index-input-test.txt"; String content = "Index input test content"; - // Write content pushContent(path, content); - // Read using BlobIndexInput try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { byte[] buffer = new byte[1024]; input.readBytes(buffer, 0, content.length()); @@ -44,16 +42,12 @@ public void testIndexInputSeek() throws Exception { String path = "index-input-seek-test.txt"; String content = "Index input seek test content"; - // Write content pushContent(path, content); - // Test seeking try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { - // Seek to middle of content long seekPosition = content.length() / 2; input.seek(seekPosition); - // Read remaining content byte[] buffer = new byte[1024]; String expectedContent = content.substring((int) seekPosition); input.readBytes(buffer, 0, expectedContent.length()); @@ -67,10 +61,8 @@ public void testIndexInputLength() throws Exception { String path = "index-input-length-test.txt"; String content = "Length test content"; - // Write content pushContent(path, content); - // Test length try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { assertEquals("Length should match", content.length(), input.length()); } @@ -81,16 +73,15 @@ public void testIndexInputReadByte() throws Exception { String path = "index-input-byte-test.txt"; String content = "Byte read test"; - // Write content pushContent(path, content); - // Test reading byte by byte try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { StringBuilder readContent = new StringBuilder(); for (int i = 0; i < content.length(); i++) { byte b = input.readByte(); readContent.append((char) b); } + assertEquals("Byte by byte content should match", content, readContent.toString()); } } @@ -100,15 +91,12 @@ public void testIndexInputReadBytes() throws Exception { String path = "index-input-bytes-test.txt"; String content = "Bytes read test content"; - // Write content pushContent(path, content); - // Test reading bytes try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { byte[] buffer = new byte[10]; StringBuilder readContent = new StringBuilder(); - // Read all content in chunks long remaining = input.length(); while (remaining > 0) { int toRead = (int) Math.min(buffer.length, remaining); @@ -126,20 +114,11 @@ public void testIndexInputSeekToEnd() throws Exception { String path = "index-input-seek-end-test.txt"; String content = "Seek to end test"; - // Write content pushContent(path, content); - // Test seeking to end try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { input.seek(content.length()); - - // Should be at end, no more bytes to read - try { - input.readByte(); - fail("Should throw EOFException when reading past end"); - } catch (IOException e) { - // Expected - } + expectThrows(IOException.class, input::readByte); } } @@ -148,17 +127,11 @@ public void testIndexInputSeekBeyondEnd() throws Exception { String path = "index-input-seek-beyond-test.txt"; String content = "Seek beyond end test"; - // Write content pushContent(path, content); - // Test seeking beyond end try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { - try { - input.seek(content.length() + 1); - fail("Should throw IOException when seeking beyond end"); - } catch (IOException e) { - // Expected - } + long invalidPosition = content.length() + 1L; + expectThrows(IOException.class, () -> input.seek(invalidPosition)); } } @@ -167,19 +140,15 @@ public void testIndexInputGetFilePointer() throws Exception { String path = "index-input-pointer-test.txt"; String content = "File pointer test content"; - // Write content pushContent(path, content); - // Test file pointer try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { assertEquals("Initial position should be 0", 0, input.getFilePointer()); - // Read some bytes byte[] buffer = new byte[5]; input.readBytes(buffer, 0, buffer.length); assertEquals("Position should be 5 after reading 5 bytes", 5, input.getFilePointer()); - // Seek to different position input.seek(10); assertEquals("Position should be 10 after seek", 10, input.getFilePointer()); } @@ -190,24 +159,18 @@ public void testIndexInputLargeFile() throws Exception { String path = "index-input-large-test.txt"; StringBuilder contentBuilder = new StringBuilder(); - // Create large content (1MB) for (int i = 0; i < 10000; i++) { contentBuilder.append("This is line ").append(i).append(" of the large file.\n"); } String content = contentBuilder.toString(); - // Write content pushContent(path, content); - // Test reading large file try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { assertEquals("Length should match", content.length(), input.length()); - // Read in chunks byte[] buffer = new byte[8192]; StringBuilder readContent = new StringBuilder(); - - // Read all content in chunks long remaining = input.length(); while (remaining > 0) { int toRead = (int) Math.min(buffer.length, remaining); @@ -225,21 +188,12 @@ public void testIndexInputEmptyFile() throws Exception { String path = "index-input-empty-test.txt"; String content = ""; - // Write empty content pushContent(path, content); - // Test reading empty file try (AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path))) { assertEquals("Length should be 0", 0, input.length()); assertEquals("Position should be 0", 0, input.getFilePointer()); - - // Should be at end immediately - try { - input.readByte(); - fail("Should throw EOFException when reading from empty file"); - } catch (IOException e) { - // Expected - } + expectThrows(IOException.class, input::readByte); } } @@ -248,27 +202,13 @@ public void testIndexInputClose() throws Exception { String path = "index-input-close-test.txt"; String content = "Close test content"; - // Write content pushContent(path, content); - // Test closing AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path)); input.close(); - // Test that operations on closed input throw exception - try { - input.readByte(); - fail("Should throw IOException when reading from closed input"); - } catch (IOException e) { - // Expected - } - - try { - input.seek(0); - fail("Should throw IOException when seeking on closed input"); - } catch (IOException e) { - // Expected - } + expectThrows(IOException.class, input::readByte); + expectThrows(IOException.class, () -> input.seek(0)); } @Test @@ -276,12 +216,10 @@ public void testIndexInputMultipleClose() throws Exception { String path = "index-input-multiple-close-test.txt"; String content = "Multiple close test content"; - // Write content pushContent(path, content); - // Test multiple close calls AzureBlobIndexInput input = new AzureBlobIndexInput(path, client, client.length(path)); input.close(); - input.close(); // Should not throw exception + input.close(); } } diff --git a/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobInstallShardTest.java b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobInstallShardTest.java index de18f10bcaf7..6ad689a81a39 100644 --- a/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobInstallShardTest.java +++ b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobInstallShardTest.java @@ -25,18 +25,15 @@ public class AzureBlobInstallShardTest extends AbstractAzureBlobClientTest { public void testInstallShard() throws Exception { String shardPath = "install-shard-test/"; - // Create shard structure client.createDirectory(shardPath); client.createDirectory(shardPath + "index/"); client.createDirectory(shardPath + "conf/"); - // Add shard files pushContent(shardPath + "index/segments_1", "Shard index segments"); pushContent(shardPath + "index/_0.cfs", "Shard index file"); pushContent(shardPath + "conf/solrconfig.xml", "Shard configuration"); pushContent(shardPath + "conf/schema.xml", "Shard schema"); - // Verify shard structure assertTrue("Shard directory should exist", client.pathExists(shardPath)); assertTrue("Index directory should exist", client.pathExists(shardPath + "index/")); assertTrue("Conf directory should exist", client.pathExists(shardPath + "conf/")); @@ -49,19 +46,15 @@ public void testInstallShard() throws Exception { @Test public void testInstallShardWithMultipleIndexFiles() throws Exception { String shardPath = "multi-index-shard-test/"; + String[] indexFiles = {"segments_1", "_0.cfs", "_0.cfe", "_0.si", "_1.cfs", "_1.cfe", "_1.si"}; - // Create shard structure client.createDirectory(shardPath); client.createDirectory(shardPath + "index/"); - // Add multiple index files - String[] indexFiles = {"segments_1", "_0.cfs", "_0.cfe", "_0.si", "_1.cfs", "_1.cfe", "_1.si"}; - for (String indexFile : indexFiles) { pushContent(shardPath + "index/" + indexFile, "Index file content: " + indexFile); } - // Verify all index files exist for (String indexFile : indexFiles) { assertTrue( "Index file should exist: " + indexFile, @@ -72,21 +65,17 @@ public void testInstallShardWithMultipleIndexFiles() throws Exception { @Test public void testInstallShardWithDataFiles() throws Exception { String shardPath = "data-shard-test/"; - - // Create shard structure - client.createDirectory(shardPath); - client.createDirectory(shardPath + "data/"); - - // Add data files String[] dataFiles = { "tlog.0000000000000000001", "tlog.0000000000000000002", "tlog.0000000000000000003" }; + client.createDirectory(shardPath); + client.createDirectory(shardPath + "data/"); + for (String dataFile : dataFiles) { pushContent(shardPath + "data/" + dataFile, "Transaction log: " + dataFile); } - // Verify all data files exist for (String dataFile : dataFiles) { assertTrue( "Data file should exist: " + dataFile, client.pathExists(shardPath + "data/" + dataFile)); @@ -96,12 +85,6 @@ public void testInstallShardWithDataFiles() throws Exception { @Test public void testInstallShardWithConfiguration() throws Exception { String shardPath = "config-shard-test/"; - - // Create shard structure - client.createDirectory(shardPath); - client.createDirectory(shardPath + "conf/"); - - // Add configuration files String solrConfig = "\n" + "\n" @@ -115,14 +98,15 @@ public void testInstallShardWithConfiguration() throws Exception { + " \n" + ""; + client.createDirectory(shardPath); + client.createDirectory(shardPath + "conf/"); + pushContent(shardPath + "conf/solrconfig.xml", solrConfig); pushContent(shardPath + "conf/schema.xml", schema); - // Verify configuration files assertTrue("Solr config should exist", client.pathExists(shardPath + "conf/solrconfig.xml")); assertTrue("Schema should exist", client.pathExists(shardPath + "conf/schema.xml")); - // Verify content try (var input = client.pullStream(shardPath + "conf/solrconfig.xml")) { byte[] buffer = new byte[1024]; int bytesRead = input.read(buffer); @@ -136,20 +120,16 @@ public void testInstallShardWithConfiguration() throws Exception { @Test public void testInstallShardWithLargeIndex() throws Exception { String shardPath = "large-index-shard-test/"; - - // Create shard structure - client.createDirectory(shardPath); - client.createDirectory(shardPath + "index/"); - - // Create large index file StringBuilder largeContent = new StringBuilder(); for (int i = 0; i < 50000; i++) { largeContent.append("Index data line ").append(i).append("\n"); } + client.createDirectory(shardPath); + client.createDirectory(shardPath + "index/"); + pushContent(shardPath + "index/large-index.cfs", largeContent.toString()); - // Verify large index file assertTrue( "Large index file should exist", client.pathExists(shardPath + "index/large-index.cfs")); assertEquals( @@ -161,20 +141,16 @@ public void testInstallShardWithLargeIndex() throws Exception { @Test public void testInstallShardWithBinaryIndex() throws Exception { String shardPath = "binary-index-shard-test/"; - - // Create shard structure - client.createDirectory(shardPath); - client.createDirectory(shardPath + "index/"); - - // Create binary index file byte[] binaryData = new byte[2048]; for (int i = 0; i < binaryData.length; i++) { binaryData[i] = (byte) (i % 256); } + client.createDirectory(shardPath); + client.createDirectory(shardPath + "index/"); + pushContent(shardPath + "index/binary-index.cfs", binaryData); - // Verify binary index file assertTrue( "Binary index file should exist", client.pathExists(shardPath + "index/binary-index.cfs")); assertEquals( @@ -187,27 +163,22 @@ public void testInstallShardWithBinaryIndex() throws Exception { public void testInstallShardWithNestedStructure() throws Exception { String shardPath = "nested-shard-test/"; - // Create nested shard structure client.createDirectory(shardPath); client.createDirectory(shardPath + "index/"); client.createDirectory(shardPath + "conf/"); client.createDirectory(shardPath + "data/"); client.createDirectory(shardPath + "logs/"); - // Add files at different levels pushContent(shardPath + "index/segments_1", "Segments file"); pushContent(shardPath + "conf/solrconfig.xml", "Config file"); pushContent(shardPath + "data/tlog.1", "Transaction log"); pushContent(shardPath + "logs/solr.log", "Log file"); - // Verify nested structure assertTrue("Root shard should exist", client.pathExists(shardPath)); assertTrue("Index directory should exist", client.pathExists(shardPath + "index/")); assertTrue("Conf directory should exist", client.pathExists(shardPath + "conf/")); assertTrue("Data directory should exist", client.pathExists(shardPath + "data/")); assertTrue("Logs directory should exist", client.pathExists(shardPath + "logs/")); - - // Verify files exist assertTrue("Segments file should exist", client.pathExists(shardPath + "index/segments_1")); assertTrue("Config file should exist", client.pathExists(shardPath + "conf/solrconfig.xml")); assertTrue("Transaction log should exist", client.pathExists(shardPath + "data/tlog.1")); @@ -217,11 +188,6 @@ public void testInstallShardWithNestedStructure() throws Exception { @Test public void testInstallShardWithMetadata() throws Exception { String shardPath = "metadata-shard-test/"; - - // Create shard structure - client.createDirectory(shardPath); - - // Add metadata files String metadata = "{\n" + " \"shardId\": \"shard1\",\n" @@ -230,14 +196,14 @@ public void testInstallShardWithMetadata() throws Exception { + " \"timestamp\": \"2023-01-01T00:00:00Z\"\n" + "}"; + client.createDirectory(shardPath); + pushContent(shardPath + "shard-metadata.json", metadata); pushContent(shardPath + "index/segments_1", "Index segments"); - // Verify metadata assertTrue("Metadata file should exist", client.pathExists(shardPath + "shard-metadata.json")); assertTrue("Index file should exist", client.pathExists(shardPath + "index/segments_1")); - // Verify metadata content try (var input = client.pullStream(shardPath + "shard-metadata.json")) { byte[] buffer = new byte[1024]; int bytesRead = input.read(buffer); @@ -251,22 +217,17 @@ public void testInstallShardWithMetadata() throws Exception { public void testInstallShardCleanup() throws Exception { String shardPath = "cleanup-shard-test/"; - // Create shard structure client.createDirectory(shardPath); client.createDirectory(shardPath + "index/"); client.createDirectory(shardPath + "conf/"); - // Add shard files pushContent(shardPath + "index/segments_1", "Index segments"); pushContent(shardPath + "conf/solrconfig.xml", "Config file"); - // Verify shard exists assertTrue("Shard should exist", client.pathExists(shardPath)); - // Cleanup shard client.deleteDirectory(shardPath); - // Verify shard is cleaned up assertFalse("Shard should not exist after cleanup", client.pathExists(shardPath)); assertFalse( "Index directory should not exist after cleanup", client.pathExists(shardPath + "index/")); diff --git a/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobOutputStreamTest.java b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobOutputStreamTest.java index 919b72d1a30d..dbfcb9a9ca5d 100644 --- a/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobOutputStreamTest.java +++ b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobOutputStreamTest.java @@ -33,7 +33,6 @@ public void testBasicOutputStream() throws Exception { output.write(content.getBytes(StandardCharsets.UTF_8)); } - // Verify content was written assertTrue("File should exist", client.pathExists(path)); try (InputStream input = client.pullStream(path)) { @@ -55,7 +54,6 @@ public void testOutputStreamWriteByte() throws Exception { } } - // Verify content was written assertTrue("File should exist", client.pathExists(path)); try (InputStream input = client.pullStream(path)) { @@ -76,7 +74,6 @@ public void testOutputStreamWriteByteArray() throws Exception { output.write(contentBytes); } - // Verify content was written assertTrue("File should exist", client.pathExists(path)); try (InputStream input = client.pullStream(path)) { @@ -99,7 +96,6 @@ public void testOutputStreamWriteByteArrayWithOffset() throws Exception { output.write(fullBytes, offset, partialContent.length()); } - // Verify content was written assertTrue("File should exist", client.pathExists(path)); try (InputStream input = client.pullStream(path)) { @@ -118,8 +114,6 @@ public void testOutputStreamFlush() throws Exception { try (OutputStream output = client.pushStream(path)) { output.write(content.getBytes(StandardCharsets.UTF_8)); output.flush(); - - // Verify content is available after flush assertTrue("File should exist after flush", client.pathExists(path)); } } @@ -133,23 +127,11 @@ public void testOutputStreamClose() throws Exception { output.write(content.getBytes(StandardCharsets.UTF_8)); output.close(); - // Verify content was written assertTrue("File should exist after close", client.pathExists(path)); - // Test that operations on closed stream throw exception - try { - output.write(1); - fail("Should throw IOException when writing to closed stream"); - } catch (IOException e) { - // Expected - } - - try { - output.flush(); - fail("Should throw IOException when flushing closed stream"); - } catch (IOException e) { - // Expected - } + OutputStream closedOutput = output; + expectThrows(IOException.class, () -> closedOutput.write(1)); + expectThrows(IOException.class, () -> closedOutput.flush()); } @Test @@ -160,9 +142,8 @@ public void testOutputStreamMultipleClose() throws Exception { OutputStream output = client.pushStream(path); output.write(content.getBytes(StandardCharsets.UTF_8)); output.close(); - output.close(); // Should not throw exception + output.close(); - // Verify content was written assertTrue("File should exist", client.pathExists(path)); } @@ -171,7 +152,6 @@ public void testOutputStreamLargeData() throws Exception { String path = "output-stream-large-test.txt"; StringBuilder contentBuilder = new StringBuilder(); - // Create large content (2MB) for (int i = 0; i < 20000; i++) { contentBuilder.append("This is line ").append(i).append(" of the large file.\n"); } @@ -181,11 +161,9 @@ public void testOutputStreamLargeData() throws Exception { output.write(content.getBytes(StandardCharsets.UTF_8)); } - // Verify content was written assertTrue("Large file should exist", client.pathExists(path)); assertEquals("File length should match", content.length(), client.length(path)); - // Verify content integrity try (InputStream input = client.pullStream(path)) { byte[] buffer = new byte[8192]; StringBuilder readContentBuilder = new StringBuilder(); @@ -204,7 +182,6 @@ public void testOutputStreamChunkedWrite() throws Exception { byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); try (OutputStream output = client.pushStream(path)) { - // Write in small chunks int chunkSize = 5; for (int i = 0; i < contentBytes.length; i += chunkSize) { int remaining = Math.min(chunkSize, contentBytes.length - i); @@ -212,7 +189,6 @@ public void testOutputStreamChunkedWrite() throws Exception { } } - // Verify content was written correctly assertTrue("File should exist", client.pathExists(path)); try (InputStream input = client.pullStream(path)) { @@ -228,7 +204,6 @@ public void testOutputStreamBinaryData() throws Exception { String path = "output-stream-binary-test.bin"; byte[] binaryData = new byte[1024]; - // Fill with some binary data for (int i = 0; i < binaryData.length; i++) { binaryData[i] = (byte) (i % 256); } @@ -237,11 +212,9 @@ public void testOutputStreamBinaryData() throws Exception { output.write(binaryData); } - // Verify binary data was written assertTrue("Binary file should exist", client.pathExists(path)); assertEquals("Binary file length should match", binaryData.length, client.length(path)); - // Verify binary data integrity try (InputStream input = client.pullStream(path)) { byte[] readData = new byte[binaryData.length]; int bytesRead = input.read(readData); diff --git a/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobPathsTest.java b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobPathsTest.java index 787038dea440..2991340f868a 100644 --- a/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobPathsTest.java +++ b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobPathsTest.java @@ -24,13 +24,10 @@ public class AzureBlobPathsTest extends AbstractAzureBlobClientTest { public void testPathExists() throws Exception { String path = "path-exists-test-" + java.util.UUID.randomUUID() + ".txt"; - // Initially should not exist assertFalse("Path should not exist initially", client.pathExists(path)); - // Create file pushContent(path, "test content"); - // Should exist now assertTrue("Path should exist after creation", client.pathExists(path)); } @@ -38,13 +35,10 @@ public void testPathExists() throws Exception { public void testDirectoryExists() throws Exception { String dirPath = "test-directory-" + java.util.UUID.randomUUID() + "/"; - // Initially should not exist assertFalse("Directory should not exist initially", client.pathExists(dirPath)); - // Create directory client.createDirectory(dirPath); - // Should exist now assertTrue("Directory should exist after creation", client.pathExists(dirPath)); } @@ -53,11 +47,9 @@ public void testIsDirectory() throws Exception { String dirPath = "is-directory-test/"; String filePath = "is-directory-test.txt"; - // Create directory client.createDirectory(dirPath); assertTrue("Should be a directory", client.isDirectory(dirPath)); - // Create file pushContent(filePath, "test content"); assertFalse("Should not be a directory", client.isDirectory(filePath)); } @@ -67,10 +59,8 @@ public void testFileLength() throws Exception { String path = "file-length-test.txt"; String content = "File length test content"; - // Create file pushContent(path, content); - // Check length assertEquals("File length should match", content.length(), client.length(path)); } @@ -78,30 +68,20 @@ public void testFileLength() throws Exception { public void testDirectoryLength() throws Exception { String dirPath = "directory-length-test/"; - // Create directory client.createDirectory(dirPath); - // Should throw exception when getting length of directory - try { - client.length(dirPath); - fail("Should throw exception when getting length of directory"); - } catch (AzureBlobException e) { - // Expected - } + expectThrows(AzureBlobException.class, () -> client.length(dirPath)); } @Test public void testListDirectory() throws Exception { String dirPath = "list-directory-test/"; - // Create directory client.createDirectory(dirPath); - // Initially should be empty String[] files = client.listDir(dirPath); assertEquals("Directory should be empty initially", 0, files.length); - // Add some files String[] fileNames = {"file1.txt", "file2.txt", "subdir/"}; for (String fileName : fileNames) { String fullPath = dirPath + fileName; @@ -112,11 +92,9 @@ public void testListDirectory() throws Exception { } } - // List directory contents files = client.listDir(dirPath); assertEquals("Should list all files and directories", fileNames.length, files.length); - // Verify all files are listed for (String fileName : fileNames) { boolean found = false; for (String listedFile : files) { @@ -133,7 +111,6 @@ public void testListDirectory() throws Exception { public void testListAll() throws Exception { String dirPath = "list-all-test/"; - // Create directory structure client.createDirectory(dirPath); client.createDirectory(dirPath + "subdir1/"); client.createDirectory(dirPath + "subdir2/"); @@ -143,11 +120,9 @@ public void testListAll() throws Exception { pushContent(dirPath + "subdir1/file3.txt", "Content 3"); pushContent(dirPath + "subdir2/file4.txt", "Content 4"); - // List all files recursively java.util.Set allFiles = new java.util.HashSet<>(); listAllRecursive(dirPath, allFiles); - // Should find all files assertTrue("Should find file1.txt", allFiles.contains(dirPath + "file1.txt")); assertTrue("Should find file2.txt", allFiles.contains(dirPath + "file2.txt")); assertTrue("Should find subdir1/file3.txt", allFiles.contains(dirPath + "subdir1/file3.txt")); @@ -174,14 +149,11 @@ private void listAllRecursive(String dirPath, java.util.Set allFiles) public void testDeleteFile() throws Exception { String path = "delete-file-test.txt"; - // Create file pushContent(path, "test content"); assertTrue("File should exist", client.pathExists(path)); - // Delete file client.delete(java.util.Set.of(path)); - // Should not exist anymore assertFalse("File should not exist after deletion", client.pathExists(path)); } @@ -190,17 +162,14 @@ public void testDeleteDirectory() throws Exception { String dirPath = "delete-directory-test/"; String filePath = dirPath + "nested-file.txt"; - // Create directory and file client.createDirectory(dirPath); pushContent(filePath, "nested content"); assertTrue("Directory should exist", client.pathExists(dirPath)); assertTrue("File should exist", client.pathExists(filePath)); - // Delete directory client.deleteDirectory(dirPath); - // Should not exist anymore assertFalse("Directory should not exist after deletion", client.pathExists(dirPath)); assertFalse("File should not exist after deletion", client.pathExists(filePath)); } @@ -209,10 +178,8 @@ public void testDeleteDirectory() throws Exception { public void testDeleteNonExistentFile() throws Exception { String path = "non-existent-file.txt"; - // Should not exist assertFalse("File should not exist", client.pathExists(path)); - // Delete non-existent file should not throw exception client.delete(java.util.Set.of(path)); } @@ -220,10 +187,8 @@ public void testDeleteNonExistentFile() throws Exception { public void testDeleteNonExistentDirectory() throws Exception { String dirPath = "non-existent-directory/"; - // Should not exist assertFalse("Directory should not exist", client.pathExists(dirPath)); - // Delete non-existent directory should not throw exception client.deleteDirectory(dirPath); } @@ -234,24 +199,20 @@ public void testNestedDirectories() throws Exception { String subDir2 = rootDir + "subdir2/"; String deepDir = subDir1 + "deepdir/"; - // Create nested directory structure client.createDirectory(rootDir); client.createDirectory(subDir1); client.createDirectory(subDir2); client.createDirectory(deepDir); - // Verify all directories exist assertTrue("Root directory should exist", client.pathExists(rootDir)); assertTrue("Sub directory 1 should exist", client.pathExists(subDir1)); assertTrue("Sub directory 2 should exist", client.pathExists(subDir2)); assertTrue("Deep directory should exist", client.pathExists(deepDir)); - // Add files to different levels pushContent(rootDir + "root-file.txt", "Root file content"); pushContent(subDir1 + "sub-file.txt", "Sub file content"); pushContent(deepDir + "deep-file.txt", "Deep file content"); - // Verify files exist assertTrue("Root file should exist", client.pathExists(rootDir + "root-file.txt")); assertTrue("Sub file should exist", client.pathExists(subDir1 + "sub-file.txt")); assertTrue("Deep file should exist", client.pathExists(deepDir + "deep-file.txt")); @@ -259,7 +220,6 @@ public void testNestedDirectories() throws Exception { @Test public void testPathSanitization() throws Exception { - // Test various path formats String[] testPaths = { "simple-file.txt", "/leading-slash.txt", @@ -272,61 +232,42 @@ public void testPathSanitization() throws Exception { }; for (String testPath : testPaths) { - try { - String sanitizedPath = client.sanitizedPath(testPath); - assertNotNull("Sanitized path should not be null", sanitizedPath); - assertFalse("Sanitized path should not start with slash", sanitizedPath.startsWith("/")); - } catch (AzureBlobException e) { - // Some paths might be invalid, which is expected - } + String sanitizedPath = client.sanitizedPath(testPath); + assertNotNull("Sanitized path should not be null", sanitizedPath); + assertFalse("Sanitized path should not start with slash", sanitizedPath.startsWith("/")); } } @Test public void testFilePathSanitization() throws Exception { - // Test file path sanitization String[] validFilePaths = { "simple-file.txt", "nested/path/file.txt", "file-with-dashes.txt", "file_with_underscores.txt" }; for (String filePath : validFilePaths) { - try { - String sanitizedPath = client.sanitizedFilePath(filePath); - assertNotNull("Sanitized file path should not be null", sanitizedPath); - assertFalse("Sanitized file path should not end with slash", sanitizedPath.endsWith("/")); - } catch (AzureBlobException e) { - fail("Valid file path should not throw exception: " + filePath); - } + String sanitizedPath = client.sanitizedFilePath(filePath); + assertNotNull("Sanitized file path should not be null", sanitizedPath); + assertFalse("Sanitized file path should not end with slash", sanitizedPath.endsWith("/")); } - // Test invalid file paths String[] invalidFilePaths = {"file-with-trailing-slash/", "", " "}; for (String filePath : invalidFilePaths) { - try { - client.sanitizedFilePath(filePath); - fail("Invalid file path should throw exception: " + filePath); - } catch (AzureBlobException e) { - // Expected - } + final String path = filePath; + expectThrows(AzureBlobException.class, () -> client.sanitizedFilePath(path)); } } @Test public void testDirectoryPathSanitization() throws Exception { - // Test directory path sanitization String[] testDirPaths = { "simple-dir", "nested/path/dir", "dir-with-dashes", "dir_with_underscores" }; for (String dirPath : testDirPaths) { - try { - String sanitizedPath = client.sanitizedDirPath(dirPath); - assertNotNull("Sanitized directory path should not be null", sanitizedPath); - assertTrue("Sanitized directory path should end with slash", sanitizedPath.endsWith("/")); - } catch (AzureBlobException e) { - fail("Valid directory path should not throw exception: " + dirPath); - } + String sanitizedPath = client.sanitizedDirPath(dirPath); + assertNotNull("Sanitized directory path should not be null", sanitizedPath); + assertTrue("Sanitized directory path should end with slash", sanitizedPath.endsWith("/")); } } } diff --git a/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobReadWriteTest.java b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobReadWriteTest.java index 370fe7321d29..33f0a2177855 100644 --- a/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobReadWriteTest.java +++ b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AzureBlobReadWriteTest.java @@ -17,7 +17,6 @@ package org.apache.solr.azureblob; import com.carrotsearch.randomizedtesting.generators.RandomBytes; -import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; @@ -30,10 +29,8 @@ public void testBasicReadWrite() throws Exception { String path = "test-file.txt"; String content = "Hello, Azure Blob Storage!"; - // Write content pushContent(path, content); - // Read content try (InputStream input = client.pullStream(path)) { byte[] buffer = new byte[1024]; int bytesRead = input.read(buffer); @@ -47,20 +44,16 @@ public void testLargeFileReadWrite() throws Exception { String path = "large-file.txt"; StringBuilder contentBuilder = new StringBuilder(); - // Create a large content (1MB) for (int i = 0; i < 10000; i++) { contentBuilder.append("This is line ").append(i).append(" of the large file.\n"); } String content = contentBuilder.toString(); - // Write content pushContent(path, content); - // Verify file exists and has correct length assertTrue("File should exist", client.pathExists(path)); assertEquals("File length should match", content.length(), client.length(path)); - // Read content back try (InputStream input = client.pullStream(path)) { byte[] buffer = new byte[8192]; StringBuilder readContentBuilder = new StringBuilder(); @@ -77,15 +70,12 @@ public void testBinaryDataReadWrite() throws Exception { String path = "binary-file.bin"; byte[] binaryData = new byte[1024]; - // Fill with some binary data for (int i = 0; i < binaryData.length; i++) { binaryData[i] = (byte) (i % 256); } - // Write binary data pushContent(path, binaryData); - // Read binary data back try (InputStream input = client.pullStream(path)) { byte[] readData = new byte[binaryData.length]; int bytesRead = input.read(readData); @@ -102,10 +92,8 @@ public void testConcurrentReadWrite() throws Exception { String path = "concurrent-file.txt"; String content = "Concurrent read/write test content"; - // Write content pushContent(path, content); - // Read from multiple streams concurrently try (InputStream input1 = client.pullStream(path); InputStream input2 = client.pullStream(path)) { @@ -128,22 +116,17 @@ public void testStreamClose() throws Exception { String path = "stream-close-test.txt"; String content = "Stream close test content"; - // Write content pushContent(path, content); - // Test that stream can be closed multiple times without exception InputStream input = client.pullStream(path); input.close(); - input.close(); // Should not throw exception + input.close(); - // ResumableInputStream automatically resumes after close, so we can still read - // This tests the resumable behavior - a new stream is created on read int firstByte = input.read(); assertTrue( "Stream should be resumable after close (got byte: " + firstByte + ")", - firstByte >= 0 || firstByte == -1); // Either valid byte or EOF + firstByte >= 0 || firstByte == -1); - // Close again after successful resume input.close(); } @@ -152,14 +135,11 @@ public void testEmptyFileReadWrite() throws Exception { String path = "empty-file.txt"; String content = ""; - // Write empty content pushContent(path, content); - // Verify file exists assertTrue("Empty file should exist", client.pathExists(path)); assertEquals("Empty file should have zero length", 0, client.length(path)); - // Read empty content try (InputStream input = client.pullStream(path)) { int bytesRead = input.read(); assertEquals("Should return -1 for empty file", -1, bytesRead); @@ -171,10 +151,8 @@ public void testUnicodeContentReadWrite() throws Exception { String path = "unicode-file.txt"; String content = "Hello 世界! 🌍 Unicode test: αβγδε"; - // Write Unicode content pushContent(path, content); - // Read Unicode content back try (InputStream input = client.pullStream(path)) { byte[] buffer = new byte[1024]; int bytesRead = input.read(buffer); @@ -188,13 +166,11 @@ public void testOutputStreamFlush() throws Exception { String path = "flush-test.txt"; String content = "Flush test content"; - // Write content with explicit flush try (OutputStream output = client.pushStream(path)) { output.write(content.getBytes(StandardCharsets.UTF_8)); output.flush(); } - // Verify content was written assertTrue("File should exist after flush", client.pathExists(path)); try (InputStream input = client.pullStream(path)) { @@ -209,12 +185,11 @@ public void testOutputStreamFlush() throws Exception { public void testReadWithConnectionLoss() throws Exception { String key = "flush-very-large"; - int numBytes = 2_000_000; // keep this small to avoid long retries with Azure client + int numBytes = 2_000_000; pushContent(key, RandomBytes.randomBytesOfLength(random(), numBytes)); - int numExceptions = 5; // fewer induced failures for Azure path + int numExceptions = 5; int bytesPerException = numBytes / numExceptions; - // Check we can re-read same content int maxBuffer = 100; byte[] buffer = new byte[maxBuffer]; @@ -223,11 +198,8 @@ public void testReadWithConnectionLoss() throws Exception { long byteCount = 0; long lastResetBucket = -1; while (!done) { - // Use the same number of bytes no matter which method we are testing int numBytesToRead = random().nextInt(maxBuffer) + 1; - // test both read() and read(buffer, off, len) switch (random().nextInt(3)) { - // read() case 0: { for (int i = 0; i < numBytesToRead && !done; i++) { @@ -238,43 +210,36 @@ public void testReadWithConnectionLoss() throws Exception { } } break; - // read(byte, off, len) case 1: { int readLen = input.read(buffer, 0, numBytesToRead); if (readLen > 0) { byteCount += readLen; } else { - // We are done when readLen = -1 done = true; } } break; - // skip(len) case 2: { - // We only want to skip 1 because long bytesSkipped = input.skip(numBytesToRead); byteCount += bytesSkipped; if (bytesSkipped < numBytesToRead) { - // We are done when no bytes are skipped done = true; } } break; } + // Initiate a connection loss at the beginning of every "bytesPerException" cycle. // The input stream will not immediately see an error, it will have pre-loaded some data. long currentBucket = byteCount / bytesPerException; if (currentBucket != lastResetBucket && (byteCount % bytesPerException <= maxBuffer)) { - try { - initiateBlobConnectionLoss(); - } catch (AzureBlobException e) { - throw new IOException("Failed to simulate connection loss", e); - } + initiateBlobConnectionLoss(); lastResetBucket = currentBucket; } } + assertEquals("Wrong amount of data found from InputStream", numBytes, byteCount); } } diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc index d56a458db21e..28b2a0f4c7f7 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc @@ -383,7 +383,7 @@ If the status is anything other than "success", an error message will explain wh Solr provides a repository abstraction to allow users to backup and restore their data to a variety of different storage systems. For example, a Solr cluster running on a local filesystem (e.g., EXT3) can store backup data on the same disk, on a remote network-mounted drive, or in some popular "cloud storage" providers, depending on the 'repository' implementation chosen. -Solr offers multiple different repository implementations out of the box (`LocalFileSystemRepository`, `GCSBackupRepository`, `S3BackupRepository`, and `BlobBackupRepository`), and allows users to create plugins for their own storage systems as needed. It is also possible to create a `DelegatingBackupRepository` that delegates to another `BackupRepository` and adds or modifies some behavior on top of it. +Solr offers multiple different repository implementations out of the box (`LocalFileSystemRepository`, `GCSBackupRepository`, `S3BackupRepository`, and `AzureBlobBackupRepository`), and allows users to create plugins for their own storage systems as needed. It is also possible to create a `DelegatingBackupRepository` that delegates to another `BackupRepository` and adds or modifies some behavior on top of it. Users can define any number of repositories in their `solr.xml` file. The backup and restore APIs described above allow users to select which of these definitions they want to use at runtime via the `repository` parameter. @@ -795,21 +795,15 @@ https://docs.aws.amazon.com/sdkref/latest/guide/settings-global.html[These optio ** RetryMode (`LEGACY`, `STANDARD`, `ADAPTIVE`) ** Max Attempts -=== BlobBackupRepository +=== AzureBlobBackupRepository Stores and retrieves backup files in a Microsoft Azure Blob Storage container. This is provided via the `azure-blob-repository` xref:configuration-guide:solr-modules.adoc[Solr Module] that needs to be enabled before use. -AzureBlobBackupRepository supports four authentication methods, each suitable for different deployment scenarios: +This plugin supports multiple authentication methods: connection strings, account keys, SAS tokens, and Azure Identity (Managed Identity, Service Principal, Azure CLI). +For Azure Identity, ensure the identity has the "Storage Blob Data Contributor" role on the storage account. -==== Authentication Methods - -*Connection String* (recommended for development/testing):: -+ -The simplest authentication method using a complete Azure Storage connection string. -Ideal for local development with Azurite emulator or quick testing. -+ [source,xml] ---- @@ -820,90 +814,7 @@ Ideal for local development with Azurite emulator or quick testing. ---- -*Account Name + Access Key* (recommended for simple production):: -+ -Separates the account name from the access key, providing cleaner configuration and easier credential rotation. -+ -[source,xml] ----- - - - solr-backup - myaccount - mykey - - ----- - -*Shared Access Signature (SAS) Token* (recommended for production with time-limited access):: -+ -Provides time-limited, permission-scoped access without exposing account keys. -SAS tokens must include service, container, and object permissions (`srt=sco`) with read, write, delete, list, add, and create permissions (`sp=rwdlac`). -+ -The container must be pre-created before using a SAS token. -+ -[source,xml] ----- - - - solr-backup - myaccount - sv=2024-11-04&ss=b&srt=sco&sp=rwdlacytfx&se=2025-12-31T23:59:59Z&st=2025-01-01T00:00:00Z&spr=https&sig=... - - ----- -+ -NOTE: SAS tokens in XML must have `&` characters escaped as `&`. - -*Azure Identity* (recommended for production on Azure infrastructure):: -+ -Uses Azure Active Directory (Azure Entra ID) authentication, supporting Managed Identities, Service Principals, and Azure CLI credentials. -This is the most secure option for production deployments running on Azure infrastructure. -+ -For *Managed Identity* (for VMs, AKS, App Service): -+ -[source,xml] ----- - - - solr-backup - https://myaccount.blob.core.windows.net - - ----- -+ -For *Service Principal*: -+ -[source,xml] ----- - - - solr-backup - https://myaccount.blob.core.windows.net - your-tenant-id - your-client-id - your-client-secret - - ----- -+ -For *Azure CLI* (development only): -+ -[source,xml] ----- - - - solr-backup - https://myaccount.blob.core.windows.net - - ----- -+ -NOTE: When using Azure Identity, the identity must have the "Storage Blob Data Contributor" role assigned to the storage account. - -==== Configuration Options - -AzureBlobBackupRepository accepts the following configuration options: +AzureBlobBackupRepository accepts the following options for configuration: `azure.blob.container.name`:: + @@ -912,8 +823,7 @@ AzureBlobBackupRepository accepts the following configuration options: |Required |Default: none |=== + -The name of the Azure Blob Storage container to use for backups. -The container must exist before performing backup operations. +The name of the Azure Blob Storage container. The container must exist before performing backup operations. `azure.blob.connection.string`:: + @@ -922,9 +832,7 @@ The container must exist before performing backup operations. |Optional |Default: none |=== + -Complete Azure Storage connection string including account name, key, and endpoints. -Required for Connection String authentication. -Mutually exclusive with other authentication methods. +Complete Azure Storage connection string. Mutually exclusive with other authentication methods. `azure.blob.account.name`:: + @@ -933,8 +841,7 @@ Mutually exclusive with other authentication methods. |Optional |Default: none |=== + -Azure Storage account name. -Required for Account Name + Key and SAS Token authentication methods. +Azure Storage account name. Used with account key or SAS token authentication. `azure.blob.account.key`:: + @@ -943,9 +850,7 @@ Required for Account Name + Key and SAS Token authentication methods. |Optional |Default: none |=== + -Azure Storage account access key. -Required for Account Name + Key authentication. -Mutually exclusive with SAS token and Azure Identity. +Azure Storage account access key. Mutually exclusive with SAS token and Azure Identity. `azure.blob.sas.token`:: + @@ -954,10 +859,8 @@ Mutually exclusive with SAS token and Azure Identity. |Optional |Default: none |=== + -Shared Access Signature token for time-limited, permission-scoped access. -Must include `srt=sco` (service, container, object) and `sp=rwdlac` permissions. -The `&` characters must be XML-escaped as `&` in `solr.xml`. -Mutually exclusive with account key and Azure Identity. +SAS token for time-limited access. Must include `srt=sco` and `sp=rwdlac` permissions. +The `&` characters must be XML-escaped as `&`. `azure.blob.endpoint`:: + @@ -966,9 +869,8 @@ Mutually exclusive with account key and Azure Identity. |Optional |Default: none |=== + -Azure Blob Storage endpoint URL in the format `https://.blob.core.windows.net`. +Azure Blob Storage endpoint URL (e.g., `https://myaccount.blob.core.windows.net`). Required for Azure Identity authentication. -Can be used with other methods to override default endpoint. `azure.blob.tenant.id`:: + @@ -977,8 +879,7 @@ Can be used with other methods to override default endpoint. |Optional |Default: none |=== + -Azure Active Directory tenant ID. -Required for Service Principal authentication. +Azure AD tenant ID for Service Principal authentication. `azure.blob.client.id`:: + @@ -987,8 +888,7 @@ Required for Service Principal authentication. |Optional |Default: none |=== + -Azure Active Directory application (client) ID. -Required for Service Principal authentication. +Azure AD application (client) ID for Service Principal authentication. `azure.blob.client.secret`:: + @@ -997,8 +897,7 @@ Required for Service Principal authentication. |Optional |Default: none |=== + -Azure Active Directory application (client) secret. -Required for Service Principal authentication. +Azure AD application secret for Service Principal authentication. `location`:: + @@ -1007,40 +906,4 @@ Required for Service Principal authentication. |Optional |Default: none |=== + -A default path prefix within the container for backup storage. -Used as a fallback when users don't provide a `location` parameter in their Backup or Restore API commands. -Can be `/` to use the root of the container. - -==== Local Development with Azurite - -For local development and testing, BlobBackupRepository works with the Azurite emulator, which provides a local Azure Storage-compatible environment. - -Install and start Azurite: -[source,bash] ----- -npm install -g azurite -azurite --blobPort 10000 ----- - -Configure `solr.xml` with Azurite connection string: -[source,xml] ----- - - - solr-backup - DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1; - - ----- - -==== Production Deployment Best Practices - -* *Use Azure Identity (Managed Identity)* for production deployments on Azure VMs, AKS, or App Service -* *Use SAS tokens* for production deployments outside Azure or when time-limited access is required -* *Avoid Connection String and Account Keys* in production as they provide unlimited access -* *Enable soft delete* on your Azure Storage account for data protection -* *Use lifecycle management* to automatically archive or delete old backups -* *Monitor backup operations* through Azure Storage metrics and logs -* *Test restore operations* regularly to ensure backup integrity - -For more detailed information on Azure authentication setup, SAS token generation, and troubleshooting, refer to the module documentation in `solr/modules/azure-blob-repository/README.md`. +Default path prefix within the container for backup storage. From 70ed112c6d99a42ab876105cf680fe4332310fbd Mon Sep 17 00:00:00 2001 From: Prateek Singhal Date: Thu, 30 Apr 2026 14:11:01 -0700 Subject: [PATCH 5/5] SOLR-17949: AGENTS.md compliance + fix SocketProxy import after upstream merge Made-with: Cursor --- gradle/libs.versions.toml | 1 + .../azure-blob-repository/build.gradle | 4 +- .../azure-blob-repository/gradle.lockfile | 207 ++++++++++++++++++ .../AbstractAzureBlobClientTest.java | 8 +- 4 files changed, 214 insertions(+), 6 deletions(-) create mode 100644 solr/modules/azure-blob-repository/gradle.lockfile diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 398dabf202f7..5f9141d67355 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -304,6 +304,7 @@ azure-core = { module = "com.azure:azure-core", version.ref = "azure-core" } azure-core-http-okhttp = { module = "com.azure:azure-core-http-okhttp", version.ref = "azure-core-http-okhttp" } azure-identity = { module = "com.azure:azure-identity", version.ref = "azure-identity" } azure-storage-blob = { module = "com.azure:azure-storage-blob", version.ref = "azure-storage" } +azure-storage-common = { module = "com.azure:azure-storage-common", version.ref = "azure-storage" } bc-jose4j = { module = "org.bitbucket.b_c:jose4j", version.ref = "bc-jose4j" } benmanes-caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version.ref = "benmanes-caffeine" } bouncycastle-bcpkix = { module = "org.bouncycastle:bcpkix-jdk18on", version.ref = "bouncycastle" } diff --git a/solr/modules/azure-blob-repository/build.gradle b/solr/modules/azure-blob-repository/build.gradle index df679db2c4b0..62f40b16f331 100644 --- a/solr/modules/azure-blob-repository/build.gradle +++ b/solr/modules/azure-blob-repository/build.gradle @@ -43,7 +43,7 @@ dependencies { exclude group: 'com.azure', module: 'azure-core-http-netty' } implementation libs.azure.core.http.okhttp - implementation('com.azure:azure-storage-common:12.25.0') { + implementation(libs.azure.storage.common) { exclude group: 'com.azure', module: 'azure-core-http-netty' } @@ -64,6 +64,6 @@ dependencies { testImplementation libs.testcontainers // Explicit transitive test dependencies for dependency analyzer - testImplementation 'com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.3' + testImplementation libs.carrotsearch.randomizedtesting.runner testImplementation libs.apache.lucene.testframework } \ No newline at end of file diff --git a/solr/modules/azure-blob-repository/gradle.lockfile b/solr/modules/azure-blob-repository/gradle.lockfile new file mode 100644 index 000000000000..01e50d22eb64 --- /dev/null +++ b/solr/modules/azure-blob-repository/gradle.lockfile @@ -0,0 +1,207 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.azure:azure-core-http-okhttp:1.13.2=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.azure:azure-core:1.57.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.azure:azure-identity:1.12.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.azure:azure-json:1.5.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.azure:azure-storage-blob:12.25.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.azure:azure-storage-common:12.25.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.azure:azure-storage-internal-avro:12.10.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.azure:azure-xml:1.2.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.4=jarValidation,testCompileClasspath,testRuntimeClasspath +com.carrotsearch:hppc:0.10.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-annotations:2.21=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-core:2.21.2=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-databind:2.21.2=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.21.2=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.21.2=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.21.2=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.module:jackson-module-jakarta-xmlbind-annotations:2.21.2=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +com.fasterxml.jackson:jackson-bom:2.21.2=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +com.fasterxml.woodstox:woodstox-core:7.0.0=apiHelper +com.fasterxml.woodstox:woodstox-core:7.1.1=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,apiHelper,errorprone,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testAnnotationProcessor,testRuntimeClasspath +com.github.docker-java:docker-java-api:3.7.0=jarValidation,testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-transport-zerodep:3.7.0=jarValidation,testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-transport:3.7.0=jarValidation,testCompileClasspath,testRuntimeClasspath +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,errorprone,testAnnotationProcessor +com.github.stephenc.jcip:jcip-annotations:1.0-1=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,errorprone,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.11.0=annotationProcessor,errorprone,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,errorprone,testAnnotationProcessor +com.google.errorprone:error_prone_annotation:2.41.0=annotationProcessor,errorprone,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +com.google.errorprone:error_prone_annotations:2.43.0=annotationProcessor,errorprone,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.41.0=annotationProcessor,errorprone,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.41.0=annotationProcessor,errorprone,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.27.0=annotationProcessor,errorprone,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,apiHelper,compileClasspath,errorprone,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +com.google.guava:guava:33.5.0-jre=annotationProcessor,apiHelper,compileClasspath,errorprone,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,apiHelper,compileClasspath,errorprone,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,apiHelper,compileClasspath,errorprone,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +com.google.protobuf:protobuf-java:3.25.8=annotationProcessor,errorprone,testAnnotationProcessor +com.j256.simplemagic:simplemagic:1.17=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +com.jayway.jsonpath:json-path:2.9.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +com.lmax:disruptor:4.0.0=solrPlatformLibs +com.microsoft.azure:msal4j-persistence-extension:1.3.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.microsoft.azure:msal4j:1.15.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.nimbusds:content-type:2.3=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.nimbusds:lang-tag:1.7=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.nimbusds:nimbus-jose-jwt:10.5=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.nimbusds:oauth2-oidc-sdk:11.9.1=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.squareup.okhttp3:okhttp:4.12.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.squareup.okio:okio-jvm:3.16.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +com.tdunning:t-digest:3.3=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +commons-cli:commons-cli:1.11.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +commons-codec:commons-codec:1.21.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.21.0=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +io.dropwizard.metrics:metrics-annotation:4.2.33=jarValidation,testRuntimeClasspath +io.dropwizard.metrics:metrics-core:4.2.33=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +io.dropwizard.metrics:metrics-jetty12-ee10:4.2.33=jarValidation,testRuntimeClasspath +io.dropwizard.metrics:metrics-jetty12:4.2.33=jarValidation,testRuntimeClasspath +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,errorprone,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,errorprone,testAnnotationProcessor +io.netty:netty-buffer:4.2.12.Final=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +io.netty:netty-codec-base:4.2.12.Final=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +io.netty:netty-common:4.2.12.Final=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +io.netty:netty-handler:4.2.12.Final=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +io.netty:netty-resolver:4.2.12.Final=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +io.netty:netty-tcnative-boringssl-static:2.0.75.Final=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +io.netty:netty-tcnative-classes:2.0.75.Final=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +io.netty:netty-transport-classes-epoll:4.2.12.Final=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +io.netty:netty-transport-native-epoll:4.2.12.Final=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +io.netty:netty-transport-native-unix-common:4.2.12.Final=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +io.netty:netty-transport:4.2.12.Final=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api-incubator:2.22.0-alpha=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:2.22.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java17:2.22.0-alpha=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java8:2.22.0-alpha=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +io.opentelemetry.semconv:opentelemetry-semconv:1.37.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +io.opentelemetry:opentelemetry-api-incubator:1.56.0-alpha=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +io.opentelemetry:opentelemetry-api:1.56.0=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +io.opentelemetry:opentelemetry-common:1.56.0=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +io.opentelemetry:opentelemetry-context:1.56.0=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +io.opentelemetry:opentelemetry-exporter-prometheus:1.56.0-alpha=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +io.opentelemetry:opentelemetry-sdk-common:1.56.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +io.opentelemetry:opentelemetry-sdk-metrics:1.56.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +io.opentelemetry:opentelemetry-sdk-trace:1.56.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +io.opentelemetry:opentelemetry-sdk:1.56.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +io.projectreactor:reactor-core:3.7.11=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +io.prometheus:prometheus-metrics-exposition-formats:1.1.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +io.prometheus:prometheus-metrics-model:1.1.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +io.sgr:s2-geometry-library-java:1.0.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +io.swagger.core.v3:swagger-annotations-jakarta:2.2.22=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +jakarta.activation:jakarta.activation-api:2.1.3=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +jakarta.annotation:jakarta.annotation-api:3.0.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +jakarta.inject:jakarta.inject-api:2.0.1=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +jakarta.servlet:jakarta.servlet-api:6.1.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +jakarta.validation:jakarta.validation-api:3.1.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +jakarta.ws.rs:jakarta.ws.rs-api:4.0.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +jakarta.xml.bind:jakarta.xml.bind-api:4.0.2=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +javax.inject:javax.inject:1=annotationProcessor,errorprone,testAnnotationProcessor +junit:junit:4.13.2=jarValidation,testCompileClasspath,testRuntimeClasspath +net.java.dev.jna:jna-platform:5.13.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +net.java.dev.jna:jna:5.18.1=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +net.minidev:accessors-smart:2.5.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +net.minidev:json-smart:2.5.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +org.antlr:antlr4-runtime:4.13.2=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.commons:commons-compress:1.28.0=jarValidation,testCompileClasspath,testRuntimeClasspath +org.apache.commons:commons-exec:1.6.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.commons:commons-lang3:3.20.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.commons:commons-math3:3.6.1=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.curator:curator-client:5.9.0=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-framework:5.9.0=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath +org.apache.httpcomponents:httpclient:4.5.14=jarValidation,testRuntimeClasspath +org.apache.httpcomponents:httpcore:4.4.16=jarValidation,testRuntimeClasspath +org.apache.httpcomponents:httpmime:4.5.14=jarValidation,testRuntimeClasspath +org.apache.logging.log4j:log4j-1.2-api:2.25.3=solrPlatformLibs +org.apache.logging.log4j:log4j-api:2.25.3=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.logging.log4j:log4j-core:2.25.3=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.logging.log4j:log4j-layout-template-json:2.25.3=solrPlatformLibs +org.apache.logging.log4j:log4j-slf4j2-impl:2.25.3=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.logging.log4j:log4j-web:2.25.3=solrPlatformLibs +org.apache.lucene:lucene-analysis-common:10.4.0=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.lucene:lucene-analysis-kuromoji:10.4.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.lucene:lucene-analysis-nori:10.4.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.lucene:lucene-analysis-phonetic:10.4.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.lucene:lucene-backward-codecs:10.4.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.lucene:lucene-classification:10.4.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.lucene:lucene-codecs:10.4.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.lucene:lucene-core:10.4.0=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.lucene:lucene-expressions:10.4.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.lucene:lucene-facet:10.4.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.lucene:lucene-grouping:10.4.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.lucene:lucene-highlighter:10.4.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.lucene:lucene-join:10.4.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.lucene:lucene-memory:10.4.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.lucene:lucene-misc:10.4.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.lucene:lucene-queries:10.4.0=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.lucene:lucene-queryparser:10.4.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.lucene:lucene-sandbox:10.4.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.lucene:lucene-spatial-extras:10.4.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.lucene:lucene-spatial3d:10.4.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.lucene:lucene-suggest:10.4.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.apache.lucene:lucene-test-framework:10.4.0=jarValidation,testCompileClasspath,testRuntimeClasspath +org.apache.zookeeper:zookeeper-jute:3.9.4=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.zookeeper:zookeeper:3.9.4=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apiguardian:apiguardian-api:1.1.2=jarValidation,testRuntimeClasspath +org.codehaus.woodstox:stax2-api:4.2.2=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.eclipse.jetty.ee10:jetty-ee10-servlet:12.0.34=jarValidation,testRuntimeClasspath +org.eclipse.jetty.http2:jetty-http2-client-transport:12.0.34=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.eclipse.jetty.http2:jetty-http2-client:12.0.34=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.eclipse.jetty.http2:jetty-http2-common:12.0.34=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.eclipse.jetty.http2:jetty-http2-hpack:12.0.34=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.eclipse.jetty.http2:jetty-http2-server:12.0.34=jarValidation,testRuntimeClasspath +org.eclipse.jetty:jetty-alpn-client:12.0.34=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.eclipse.jetty:jetty-alpn-java-client:12.0.34=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.eclipse.jetty:jetty-alpn-java-server:12.0.34=jarValidation,testRuntimeClasspath +org.eclipse.jetty:jetty-alpn-server:12.0.34=jarValidation,testRuntimeClasspath +org.eclipse.jetty:jetty-client:12.0.34=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.eclipse.jetty:jetty-http:12.0.34=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.eclipse.jetty:jetty-io:12.0.34=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.eclipse.jetty:jetty-rewrite:12.0.34=jarValidation,testRuntimeClasspath +org.eclipse.jetty:jetty-security:12.0.34=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.eclipse.jetty:jetty-server:12.0.34=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.eclipse.jetty:jetty-session:12.0.34=jarValidation,testRuntimeClasspath +org.eclipse.jetty:jetty-util:12.0.34=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.glassfish.hk2.external:aopalliance-repackaged:4.0.0-M3=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.glassfish.hk2:hk2-api:4.0.0-M3=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.glassfish.hk2:hk2-locator:4.0.0-M3=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.glassfish.hk2:hk2-utils:4.0.0-M3=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.glassfish.hk2:osgi-resource-locator:3.0.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.glassfish.jersey.containers:jersey-container-jetty-http:4.0.2=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.glassfish.jersey.core:jersey-client:4.0.2=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.glassfish.jersey.core:jersey-common:4.0.2=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.glassfish.jersey.core:jersey-server:4.0.2=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.glassfish.jersey.ext:jersey-entity-filtering:4.0.2=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.glassfish.jersey.inject:jersey-hk2:4.0.2=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.glassfish.jersey.media:jersey-media-json-jackson:4.0.2=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.glassfish.jersey:jersey-bom:4.0.2=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.hamcrest:hamcrest:3.0=jarValidation,testCompileClasspath,testRuntimeClasspath +org.javassist:javassist:3.30.2-GA=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.jetbrains.kotlin:kotlin-stdlib-jdk7:2.3.20=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +org.jetbrains.kotlin:kotlin-stdlib-jdk8:2.3.20=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +org.jetbrains.kotlin:kotlin-stdlib:2.3.20=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +org.jetbrains:annotations:26.0.2=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +org.jspecify:jspecify:1.0.0=annotationProcessor,apiHelper,compileClasspath,errorprone,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:5.6.2=jarValidation,testRuntimeClasspath +org.junit.platform:junit-platform-commons:1.6.2=jarValidation,testRuntimeClasspath +org.junit:junit-bom:5.6.2=jarValidation,testRuntimeClasspath +org.locationtech.spatial4j:spatial4j:0.8=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.opentest4j:opentest4j:1.2.0=jarValidation,testRuntimeClasspath +org.ow2.asm:asm-commons:9.8=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.ow2.asm:asm-tree:9.8=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.ow2.asm:asm:9.8=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.pcollections:pcollections:4.0.1=annotationProcessor,errorprone,testAnnotationProcessor +org.reactivestreams:reactive-streams:1.0.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +org.rnorth.duct-tape:duct-tape:1.0.8=jarValidation,testCompileClasspath,testRuntimeClasspath +org.semver4j:semver4j:6.0.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.slf4j:jcl-over-slf4j:2.0.17=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.slf4j:jul-to-slf4j:2.0.17=solrPlatformLibs +org.slf4j:slf4j-api:2.0.17=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers:2.0.3=jarValidation,testCompileClasspath,testRuntimeClasspath +org.xerial.snappy:snappy-java:1.1.10.8=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +empty=apiHelperTest,compileOnlyHelper,compileOnlyHelperTest,missingdoclet,packaging,permitAggregatorUse,permitTestAggregatorUse,permitTestUnusedDeclared,permitTestUsedUndeclared,permitUnusedDeclared,permitUsedUndeclared,signatures diff --git a/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AbstractAzureBlobClientTest.java b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AbstractAzureBlobClientTest.java index 9aaf731466f3..f6ae8f547d7c 100644 --- a/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AbstractAzureBlobClientTest.java +++ b/solr/modules/azure-blob-repository/src/test/org/apache/solr/azureblob/AbstractAzureBlobClientTest.java @@ -29,7 +29,7 @@ import okhttp3.OkHttpClient; import org.apache.lucene.tests.util.QuickPatchThreadsFilter; import org.apache.solr.SolrIgnoredThreadsFilter; -import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.SolrTestCase; import org.junit.After; import org.junit.AfterClass; import org.junit.Assume; @@ -46,7 +46,7 @@ QuickPatchThreadsFilter.class, AbstractAzureBlobClientTest.OkHttpThreadLeakFilterTest.class, }) -public class AbstractAzureBlobClientTest extends SolrTestCaseJ4 { +public class AbstractAzureBlobClientTest extends SolrTestCase { private static final String AZURITE_IMAGE = "mcr.microsoft.com/azure-storage/azurite:3.33.0"; private static final int BLOB_SERVICE_PORT = 10000; @@ -56,7 +56,7 @@ public class AbstractAzureBlobClientTest extends SolrTestCaseJ4 { private static String connectionString; protected String containerName; - protected org.apache.solr.client.solrj.cloud.SocketProxy proxy; + protected org.apache.solr.util.SocketProxy proxy; protected AzureBlobStorageClient client; @@ -84,7 +84,7 @@ public void setUpClient() throws Exception { + blobServiceUrl + "/devstoreaccount1;"; - proxy = new org.apache.solr.client.solrj.cloud.SocketProxy(); + proxy = new org.apache.solr.util.SocketProxy(); proxy.open(new java.net.URI(blobServiceUrl)); HttpClient httpClient = new OkHttpAsyncHttpClientBuilder(sharedOkHttpClient).build();