From fe3d5428f6490aaeee524ae276c7a47a5ebe7b98 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Mon, 1 Sep 2025 06:37:13 +0000 Subject: [PATCH 01/26] basic setup changes --- turbonfs/CMakeLists.txt | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index 6e8a554e..84117fcb 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -263,13 +263,14 @@ if(NOT JEMALLOC_FOUND) message(FATAL_ERROR "apt install libjemalloc-dev failed with ${JEMALLOC_INSTALL_RESULT}, try installing libjemalloc-dev manually and then run cmake again") else() # Call once more to ensure above install completed fine and also - # will set JEMALLOC_LIBRARY_DIRS variable. + # will set JEMALLOC_LIBRARIES variable. pkg_search_module(JEMALLOC REQUIRED jemalloc) endif() endif() # We want to link against the static jemalloc lib. -message(STATUS "Using jemalloc static lib ${JEMALLOC_LIBRARY_DIRS}/libjemalloc.a") +message(STATUS "Using jemalloc lib ${JEMALLOC_LIBRARIES}") +#message(STATUS "Using jemalloc static lib ${JEMALLOC_LIBRARY_DIRS}") endif() # @@ -423,14 +424,10 @@ message(STATUS "Using gnutls libraries: ${GNUTLS_ALL_LIBRARIES}") # All libraries. target_link_libraries(${CMAKE_PROJECT_NAME} - -static-libgcc -static-libstdc++ - -Wl,-rpath,/opt/microsoft/aznfs/libs ${ZLIB_LIBRARIES} dl pthread nfs - # GNUTLS libraries are needed by static libnfs. - ${GNUTLS_ALL_LIBRARIES} yaml-cpp spdlog Azure::azure-identity @@ -445,7 +442,7 @@ endif() if(ENABLE_JEMALLOC) target_link_libraries(${CMAKE_PROJECT_NAME} - ${JEMALLOC_LIBRARY_DIRS}/libjemalloc.a) + ${JEMALLOC_LIBRARIES}) endif() install(TARGETS ${CMAKE_PROJECT_NAME}) From 17d491c420b662eb211cf1f0c769ce8490b0c5d3 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Mon, 8 Sep 2025 05:58:37 +0000 Subject: [PATCH 02/26] update --- package.sh | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/package.sh b/package.sh index 55eeb252..562b704c 100755 --- a/package.sh +++ b/package.sh @@ -275,36 +275,36 @@ cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4.service ${STG_DIR}/deb/${pkg_dir}${sys # Bundle aznfsclient and its dependencies # ########################################### -# aznfsclient in the final target dir. -aznfsclient=${STG_DIR}/deb/${pkg_dir}/sbin/aznfsclient -cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} +# # aznfsclient in the final target dir. +# aznfsclient=${STG_DIR}/deb/${pkg_dir}/sbin/aznfsclient +# cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} -# Package aznfsclient dependencies in opt_dir. +# # Package aznfsclient dependencies in opt_dir. libs_dir=${STG_DIR}/deb/${pkg_dir}${opt_dir}/libs mkdir -p ${libs_dir} -# Copy the dependencies. -cp -avfH $(ldd ${aznfsclient} | grep "=>" | awk '{print $3}') ${libs_dir} - -# -# Patch all the libs to reference shared libs from ${libs_dir}. -# This is our very simple containerization. -# -for lib in ${libs_dir}/*.so*; do - echo "Setting rpath to ${opt_dir}/libs for $lib" - patchelf --set-rpath ${opt_dir}/libs "$lib" -done - -# -# Final containerization effort - bundle and use the same interpreter as the -# build machine. -# -ld_linux_path=$(ldd ${aznfsclient} | grep "ld-linux" | awk '{print $1}') -ld_linux_name=$(basename "$ld_linux_path") -ld_linux="${libs_dir}/${ld_linux_name}" -cp -avfH "${ld_linux_path}" "${ld_linux}" - -patchelf --set-interpreter ${opt_dir}/libs/${ld_linux_name} ${aznfsclient} +# # Copy the dependencies. +# cp -avfH $(ldd ${aznfsclient} | grep "=>" | awk '{print $3}') ${libs_dir} + +# # +# # Patch all the libs to reference shared libs from ${libs_dir}. +# # This is our very simple containerization. +# # +# for lib in ${libs_dir}/*.so*; do +# echo "Setting rpath to ${opt_dir}/libs for $lib" +# patchelf --set-rpath ${opt_dir}/libs "$lib" +# done + +# # +# # Final containerization effort - bundle and use the same interpreter as the +# # build machine. +# # +# ld_linux_path=$(ldd ${aznfsclient} | grep "ld-linux" | awk '{print $1}') +# ld_linux_name=$(basename "$ld_linux_path") +# ld_linux="${libs_dir}/${ld_linux_name}" +# cp -avfH "${ld_linux_path}" "${ld_linux}" + +# patchelf --set-interpreter ${opt_dir}/libs/${ld_linux_name} ${aznfsclient} # Create the deb package. dpkg-deb -Zgzip --root-owner-group --build $STG_DIR/deb/$pkg_dir From 2270de1ec26a8d15dfb03f67d1dbeb030fb7f5b4 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Mon, 8 Sep 2025 06:12:28 +0000 Subject: [PATCH 03/26] update --- package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.sh b/package.sh index 562b704c..15fe3976 100755 --- a/package.sh +++ b/package.sh @@ -276,8 +276,8 @@ cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4.service ${STG_DIR}/deb/${pkg_dir}${sys ########################################### # # aznfsclient in the final target dir. -# aznfsclient=${STG_DIR}/deb/${pkg_dir}/sbin/aznfsclient -# cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} +aznfsclient=${STG_DIR}/deb/${pkg_dir}/sbin/aznfsclient +cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} # # Package aznfsclient dependencies in opt_dir. libs_dir=${STG_DIR}/deb/${pkg_dir}${opt_dir}/libs From 391a4f6da5eba38e65a0eed9fdd8be96705f0b50 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Mon, 8 Sep 2025 06:38:54 +0000 Subject: [PATCH 04/26] update --- package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.sh b/package.sh index 15fe3976..6b060ce1 100755 --- a/package.sh +++ b/package.sh @@ -284,7 +284,7 @@ libs_dir=${STG_DIR}/deb/${pkg_dir}${opt_dir}/libs mkdir -p ${libs_dir} # # Copy the dependencies. -# cp -avfH $(ldd ${aznfsclient} | grep "=>" | awk '{print $3}') ${libs_dir} +cp -avfH $(ldd ${aznfsclient} | grep "=>" | awk '{print $3}') ${libs_dir} # # # # Patch all the libs to reference shared libs from ${libs_dir}. From e8449da3243c24c9a9b3d993012924585995b60f Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Mon, 8 Sep 2025 08:46:07 +0000 Subject: [PATCH 05/26] update --- turbonfs/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index 84117fcb..e85ebe66 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -433,7 +433,8 @@ target_link_libraries(${CMAKE_PROJECT_NAME} Azure::azure-identity Azure::azure-storage-blobs nlohmann_json::nlohmann_json - uuid) + uuid + GnuTLS::GnuTLS) if(ENABLE_TCMALLOC) target_link_libraries(${CMAKE_PROJECT_NAME} From 2de0833199421a573ff1da522c703888813a28cf Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Tue, 23 Sep 2025 10:22:34 +0000 Subject: [PATCH 06/26] update --- .github/workflows/prerelease.yml | 31 ++++++++++ .github/workflows/release.yml | 34 +++++++++++ package.sh | 99 ++++++++++++++++++++------------ packaging/aznfs/RPM/aznfs.spec | 4 ++ turbonfs/CMakeLists.txt | 71 +++++++++++++++++------ 5 files changed, 186 insertions(+), 53 deletions(-) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index ba2da2b1..c7540d51 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -55,6 +55,7 @@ jobs: export STG_DIR=$GITHUB_WORKSPACE/amd64 export SOURCE_DIR=$GITHUB_WORKSPACE/amd64 export BUILD_TYPE=${{ github.events.inputs.buildType }} + export BUILD_MACHINE=ubuntu chmod +x $SOURCE_DIR/package.sh $SOURCE_DIR/package.sh - name: Create Test Release @@ -74,6 +75,36 @@ jobs: body: | New Test Release ${{ github.event.inputs.versionName }} + package_mount_helper_azurelinux: + name: Package and Release ${{ github.event.inputs.versionName }} for azurelinux + runs-on: self-hosted + needs: verify_branch_name + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + path: azurelinux + - name: Run Package.sh for azurelinux + run: | + export RELEASE_NUMBER=${{ github.event.inputs.versionName }} + export STG_DIR=$GITHUB_WORKSPACE/azurelinux + export SOURCE_DIR=$GITHUB_WORKSPACE/azurelinux + export BUILD_TYPE=${{ github.events.inputs.buildType }} + export BUILD_MACHINE=azurelinux + chmod +x $SOURCE_DIR/package.sh + $SOURCE_DIR/package.sh + - name: Create Release (azurelinux) + uses: softprops/action-gh-release@v1 + with: + name: Release ${{ github.event.inputs.versionName }} + tag_name: ${{ github.event.inputs.versionName }} + target_commitish: ${{ github.sha }} + files: | + ${{ github.workspace }}/azurelinux/rpm/root/rpmbuild/RPMS/x86_64/aznfs-azurelinux-${{ github.event.inputs.versionName }}-1.x86_64.rpm + ${{ github.workspace }}/azurelinux/deb/aznfs-azurelinux-${{ github.event.inputs.versionName }}-1_amd64.deb + body: | + New Release ${{ github.event.inputs.versionName }} + package_mount_helper_arm64: name: Package and Release ${{ github.event.inputs.versionName }} for arm64 (Test Release) runs-on: ubuntu-22.04-arm diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3d4a592c..4021f56c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,6 +74,7 @@ jobs: export STG_DIR=$GITHUB_WORKSPACE/amd64 export SOURCE_DIR=$GITHUB_WORKSPACE/amd64 export BUILD_TYPE=${{ github.events.inputs.buildType }} + export BUILD_MACHINE=ubuntu chmod +x $SOURCE_DIR/package.sh $SOURCE_DIR/package.sh - name: Create Release @@ -92,6 +93,36 @@ jobs: body: | New Release ${{ github.event.inputs.versionName }} + package_mount_helper_azurelinux: + name: Package and Release ${{ github.event.inputs.versionName }} for azurelinux + runs-on: AzureLinux-3.0 + needs: verify_branch_name + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + path: azurelinux + - name: Run Package.sh for azurelinux + run: | + export RELEASE_NUMBER=${{ github.event.inputs.versionName }} + export STG_DIR=$GITHUB_WORKSPACE/azurelinux + export SOURCE_DIR=$GITHUB_WORKSPACE/azurelinux + export BUILD_TYPE=${{ github.events.inputs.buildType }} + export BUILD_MACHINE=azurelinux + chmod +x $SOURCE_DIR/package.sh + $SOURCE_DIR/package.sh + - name: Create Release (azurelinux) + uses: softprops/action-gh-release@v1 + with: + name: Release ${{ github.event.inputs.versionName }} + tag_name: ${{ github.event.inputs.versionName }} + target_commitish: ${{ github.sha }} + files: | + ${{ github.workspace }}/azurelinux/rpm/root/rpmbuild/RPMS/x86_64/aznfs-azurelinux-${{ github.event.inputs.versionName }}-1.x86_64.rpm + ${{ github.workspace }}/azurelinux/deb/aznfs-azurelinux-${{ github.event.inputs.versionName }}-1_amd64.deb + body: | + New Release ${{ github.event.inputs.versionName }} + package_mount_helper_arm64: name: Package and Release ${{ github.event.inputs.versionName }} for arm64 runs-on: ubuntu-22.04-arm @@ -137,3 +168,6 @@ jobs: ${{ github.workspace }}/arm64/tarball/aznfs-${{ github.event.inputs.versionName }}-1.aarch64.tar.gz body: | New Release ${{ github.event.inputs.versionName }} + + +# TODO: Add for arm64 too. \ No newline at end of file diff --git a/package.sh b/package.sh index 6b060ce1..7544830a 100755 --- a/package.sh +++ b/package.sh @@ -24,12 +24,19 @@ generate_rpm_package() { rpm_dir=$1 custom_stunnel_required=0 + azurelinux_build_required=0 # Overwrite rpm_pkg_dir in case of SUSE. if [ "$rpm_dir" == "suse" ]; then rpm_pkg_dir="${pkg_name}_sles-${RELEASE_NUMBER}-1.$arch" fi + # Overwrite rpm_pkg_dir in case of azurelinux. + if [ "$rpm_dir" == "azurelinux" ]; then + rpm_pkg_dir="${pkg_name}-azurelinux-${RELEASE_NUMBER}-1.$arch" + azurelinux_build_required=1 + fi + # Overwrite rpm_pkg_dir in case of Mariner, RedHat7, and Centos7. if [ "$rpm_dir" == "stunnel" ]; then rpm_pkg_dir="${pkg_name}_stunnel_custom-${RELEASE_NUMBER}-1.$arch" @@ -69,14 +76,17 @@ generate_rpm_package() # copy the aznfsclient binary. cp -avf ${aznfsclient} ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient - # - # Package aznfsclient dependencies in opt_dir/libs. - # libs_dir must already be populated with the required dependencies from - # the debian packaging step. Simply copy all those to rpm_libs_dir. - # - rpm_libs_dir=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/libs - mkdir -p ${rpm_libs_dir} - cp -avfH ${libs_dir}/* ${rpm_libs_dir} + + if [ "$rpm_dir" != "azurelinux" ]; then + # + # Package aznfsclient dependencies in opt_dir/libs. + # libs_dir must already be populated with the required dependencies from + # the debian packaging step. Simply copy all those to rpm_libs_dir. + # + rpm_libs_dir=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/libs + mkdir -p ${rpm_libs_dir} + cp -avfH ${libs_dir}/* ${rpm_libs_dir} + fi # Create the archive for the package. tar -cvzf ${STG_DIR}/${rpm_pkg_dir}.tar.gz -C ${STG_DIR}/${rpm_dir}/tmp root @@ -84,14 +94,16 @@ generate_rpm_package() # Copy the SPEC file to change the placeholders depending upon the RPM distro. cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/RPM/aznfs.spec ${STG_DIR}/${rpm_dir}/tmp/ - # - # Insert the contents of ${rpm_libs_dir}. - # This is variable due to the shared library versions. - # sed doesn't (easily) support replace target to be multi-line, so we use - # awk for this one. - # - opt_libs=$(for lib in ${rpm_libs_dir}/*; do echo ${opt_dir}/libs/$(basename $lib); done) - awk -i inplace -v r="$opt_libs" '{gsub(/OPT_LIBS/,r)}1' ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + if [ "$rpm_dir" != "azurelinux" ]; then + # + # Insert the contents of ${rpm_libs_dir}. + # This is variable due to the shared library versions. + # sed doesn't (easily) support replace target to be multi-line, so we use + # awk for this one. + # + opt_libs=$(for lib in ${rpm_libs_dir}/*; do echo ${opt_dir}/libs/$(basename $lib); done) + awk -i inplace -v r="$opt_libs" '{gsub(/OPT_LIBS/,r)}1' ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + fi # Insert current release number and RPM_DIR value. sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec @@ -121,7 +133,7 @@ generate_rpm_package() fi # Create the rpm package. - rpmbuild --define "custom_stunnel $custom_stunnel_required" --define "_topdir ${STG_DIR}/${rpm_dir}${rpmbuild_dir}" -v -bb ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + rpmbuild --define "custom_stunnel $custom_stunnel_required" --define "azurelinux_build $azurelinux_build_required" --define "_topdir ${STG_DIR}/${rpm_dir}${rpmbuild_dir}" -v -bb ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec } generate_tarball_package() @@ -246,6 +258,13 @@ else INSECURE_AUTH_FOR_DEVTEST=OFF fi +# Run azurelinux packaging only on azurelinux runner +if [ "$BUILD_MACHINE" == "azurelinux" ]; then + DYNAMIC_LINKS=ON +else + DYNAMIC_LINKS=OFF +fi + # vcpkg required env variable VCPKG_FORCE_SYSTEM_BINARIES to be set for arm64. if [ "$(uname -m)" == "aarch64" ]; then export VCPKG_FORCE_SYSTEM_BINARIES=1 @@ -254,6 +273,7 @@ fi cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DENABLE_PARANOID=${PARANOID} \ -DENABLE_INSECURE_AUTH_FOR_DEVTEST=${INSECURE_AUTH_FOR_DEVTEST} \ + -DENABLE_DYNAMIC_LINKS=${DYNAMIC_LINKS} \ -DPACKAGE_VERSION="${RELEASE_NUMBER}" \ -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake .. make @@ -286,25 +306,27 @@ mkdir -p ${libs_dir} # # Copy the dependencies. cp -avfH $(ldd ${aznfsclient} | grep "=>" | awk '{print $3}') ${libs_dir} -# # -# # Patch all the libs to reference shared libs from ${libs_dir}. -# # This is our very simple containerization. -# # -# for lib in ${libs_dir}/*.so*; do -# echo "Setting rpath to ${opt_dir}/libs for $lib" -# patchelf --set-rpath ${opt_dir}/libs "$lib" -# done - -# # -# # Final containerization effort - bundle and use the same interpreter as the -# # build machine. -# # -# ld_linux_path=$(ldd ${aznfsclient} | grep "ld-linux" | awk '{print $1}') -# ld_linux_name=$(basename "$ld_linux_path") -# ld_linux="${libs_dir}/${ld_linux_name}" -# cp -avfH "${ld_linux_path}" "${ld_linux}" - -# patchelf --set-interpreter ${opt_dir}/libs/${ld_linux_name} ${aznfsclient} +if [ "$BUILD_MACHINE" != "azurelinux" ]; then + # + # Patch all the libs to reference shared libs from ${libs_dir}. + # This is our very simple containerization. + # + for lib in ${libs_dir}/*.so*; do + echo "Setting rpath to ${opt_dir}/libs for $lib" + patchelf --set-rpath ${opt_dir}/libs "$lib" + done + + # + # Final containerization effort - bundle and use the same interpreter as the + # build machine. + # + ld_linux_path=$(ldd ${aznfsclient} | grep "ld-linux" | awk '{print $1}') + ld_linux_name=$(basename "$ld_linux_path") + ld_linux="${libs_dir}/${ld_linux_name}" + cp -avfH "${ld_linux_path}" "${ld_linux}" + + patchelf --set-interpreter ${opt_dir}/libs/${ld_linux_name} ${aznfsclient} +fi # Create the deb package. dpkg-deb -Zgzip --root-owner-group --build $STG_DIR/deb/$pkg_dir @@ -318,6 +340,11 @@ generate_rpm_package suse # Generate rpm package with custom stunnel installation for Mariner, RedHat7, and Centos7. generate_rpm_package stunnel +# Run azurelinux packaging only on azurelinux runner +if [ "$BUILD_MACHINE" == "azurelinux" ]; then + generate_rpm_package azurelinux +fi + ############################# # Generating Tarball for AKS# ############################# diff --git a/packaging/aznfs/RPM/aznfs.spec b/packaging/aznfs/RPM/aznfs.spec index 8b4b381e..7f7a28a3 100644 --- a/packaging/aznfs/RPM/aznfs.spec +++ b/packaging/aznfs/RPM/aznfs.spec @@ -7,6 +7,10 @@ URL: https://github.com/Azure/AZNFS-mount/blob/main/README.md %if 0%{?custom_stunnel} Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, net-tools, binutils, kernel-headers, openssl, openssl-devel, gcc, make, wget Recommends: build-essential + +%elif 0%{?mariner_build} +Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, stunnel, net-tools, libuuid, gnutls, jemalloc, libasan + %else Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, stunnel, net-tools %endif diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index e85ebe66..d01baca7 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -25,6 +25,8 @@ option(ENABLE_CHATTY "Enable super verbose logs" OFF) option(ENABLE_TCMALLOC "Use tcmalloc for malloc/free/new/delete" OFF) option(ENABLE_JEMALLOC "Use jemalloc for malloc/free/new/delete" ON) option(ENABLE_INSECURE_AUTH_FOR_DEVTEST "Enable AZAUTH for non-TLS connections" OFF) +option(ENABLE_DYNAMIC_LINKS "Use dynamic linking instead of bundling libs" OFF) + # # Builds that make it to customers need to be extra careful about any unnecessary # logging. Some warning logs we have in our code are to attract developer @@ -268,9 +270,12 @@ if(NOT JEMALLOC_FOUND) endif() endif() -# We want to link against the static jemalloc lib. -message(STATUS "Using jemalloc lib ${JEMALLOC_LIBRARIES}") -#message(STATUS "Using jemalloc static lib ${JEMALLOC_LIBRARY_DIRS}") +if(ENABLE_DYNAMIC_LINKS) + message(STATUS "Using jemalloc lib ${JEMALLOC_LIBRARIES}") +else() + # We want to link against the static jemalloc lib. + message(STATUS "Using jemalloc static lib ${JEMALLOC_LIBRARY_DIRS}") +endif() endif() # @@ -423,18 +428,37 @@ set(GNUTLS_ALL_LIBRARIES message(STATUS "Using gnutls libraries: ${GNUTLS_ALL_LIBRARIES}") # All libraries. -target_link_libraries(${CMAKE_PROJECT_NAME} - ${ZLIB_LIBRARIES} - dl - pthread - nfs - yaml-cpp - spdlog - Azure::azure-identity - Azure::azure-storage-blobs - nlohmann_json::nlohmann_json - uuid - GnuTLS::GnuTLS) +if(ENABLE_DYNAMIC_LINKS) + # Dynamic linking: No static flags, use GnuTLS::GnuTLS target + target_link_libraries(${CMAKE_PROJECT_NAME} + ${ZLIB_LIBRARIES} + dl + pthread + nfs + yaml-cpp + spdlog + Azure::azure-identity + Azure::azure-storage-blobs + nlohmann_json::nlohmann_json + uuid + GnuTLS::GnuTLS) +else() + # Static linking: Use static-libgcc/libstdc++ and explicit gnutls libraries + target_link_libraries(${CMAKE_PROJECT_NAME} + -static-libgcc -static-libstdc++ + -Wl,-rpath,/opt/microsoft/aznfs/libs + ${ZLIB_LIBRARIES} + dl + pthread + nfs + ${GNUTLS_ALL_LIBRARIES} + yaml-cpp + spdlog + Azure::azure-identity + Azure::azure-storage-blobs + nlohmann_json::nlohmann_json + uuid) +endif() if(ENABLE_TCMALLOC) target_link_libraries(${CMAKE_PROJECT_NAME} @@ -442,8 +466,21 @@ target_link_libraries(${CMAKE_PROJECT_NAME} endif() if(ENABLE_JEMALLOC) -target_link_libraries(${CMAKE_PROJECT_NAME} - ${JEMALLOC_LIBRARIES}) + if(ENABLE_DYNAMIC_LINKS) + target_link_libraries(${CMAKE_PROJECT_NAME} + ${JEMALLOC_LIBRARIES}) + else() + target_link_libraries(${CMAKE_PROJECT_NAME} + ${JEMALLOC_LIBRARY_DIRS}/libjemalloc.a) + endif() endif() +# --- Debug: Print all static libraries linked --- +get_target_property(LINK_LIBS ${CMAKE_PROJECT_NAME} LINK_LIBRARIES) +message(STATUS "==== Libraries linked to ${CMAKE_PROJECT_NAME} ====") +foreach(lib IN LISTS LINK_LIBS) + message(STATUS " ${lib}") +endforeach() +message(STATUS "===============================================") + install(TARGETS ${CMAKE_PROJECT_NAME}) From 2c128d531217740c355502bb55e9f092dc66e38d Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Tue, 23 Sep 2025 10:24:33 +0000 Subject: [PATCH 07/26] update --- .github/workflows/prerelease.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index c7540d51..a59bf0a0 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -78,7 +78,6 @@ jobs: package_mount_helper_azurelinux: name: Package and Release ${{ github.event.inputs.versionName }} for azurelinux runs-on: self-hosted - needs: verify_branch_name steps: - name: Checkout repository uses: actions/checkout@v3 From bbe469a54e5f635dcf1efbc46576fb9b8aa52c54 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Tue, 23 Sep 2025 10:48:42 +0000 Subject: [PATCH 08/26] update --- package.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/package.sh b/package.sh index 7544830a..9c896e2c 100755 --- a/package.sh +++ b/package.sh @@ -237,7 +237,12 @@ cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ # Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin +if [ "$BUILD_MACHINE" == "azurelinux" ]; then +gcc ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs +else gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs +fi +# gcc ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs # # We build the turbonfs project here, note that we can set all cmake options in the From bb23646ad325e0b0577a036af6d1bb27dbeaded1 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Tue, 23 Sep 2025 10:57:57 +0000 Subject: [PATCH 09/26] update --- .github/workflows/prerelease.yml | 4 +- package_azurelinux.sh | 187 +++++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 package_azurelinux.sh diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index a59bf0a0..17e4f26e 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -90,8 +90,8 @@ jobs: export SOURCE_DIR=$GITHUB_WORKSPACE/azurelinux export BUILD_TYPE=${{ github.events.inputs.buildType }} export BUILD_MACHINE=azurelinux - chmod +x $SOURCE_DIR/package.sh - $SOURCE_DIR/package.sh + chmod +x $SOURCE_DIR/package_azurelinux.sh + $SOURCE_DIR/package_azurelinux.sh - name: Create Release (azurelinux) uses: softprops/action-gh-release@v1 with: diff --git a/package_azurelinux.sh b/package_azurelinux.sh new file mode 100644 index 00000000..86a6ebb1 --- /dev/null +++ b/package_azurelinux.sh @@ -0,0 +1,187 @@ +# Exit on error. +set -e + +# Debian uses amd64/arm64 in place of x86_64/aarch64. +if [ "$(uname -m)" == "x86_64" ]; then + arch="x86_64" + debarch="amd64" +elif [ "$(uname -m)" == "aarch64" ]; then + arch="aarch64" + debarch="arm64" +else + echo "Unsupported architecture: $(uname -m)" + exit 1 +fi + + +generate_rpm_package() +{ + rpm_dir=$1 + custom_stunnel_required=0 + azurelinux_build_required=0 + + # Overwrite rpm_pkg_dir in case of SUSE. + if [ "$rpm_dir" == "suse" ]; then + rpm_pkg_dir="${pkg_name}_sles-${RELEASE_NUMBER}-1.$arch" + fi + + # Overwrite rpm_pkg_dir in case of azurelinux. + if [ "$rpm_dir" == "azurelinux" ]; then + rpm_pkg_dir="${pkg_name}-azurelinux-${RELEASE_NUMBER}-1.$arch" + azurelinux_build_required=1 + fi + + # Overwrite rpm_pkg_dir in case of Mariner, RedHat7, and Centos7. + if [ "$rpm_dir" == "stunnel" ]; then + rpm_pkg_dir="${pkg_name}_stunnel_custom-${RELEASE_NUMBER}-1.$arch" + custom_stunnel_required=1 + fi + + # Create the directory to hold the package spec and data files for RPM package. + mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir} + + # Copy static package file(s). + mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/usr/sbin + cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/usr/sbin/ + cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/usr/sbin/ + + # Compile mount.aznfs.c and put the executable into ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin. + mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin + gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/mount.aznfs + + mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir} + cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/mountscript.sh ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/nfsv3mountscript.sh ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/nfsv4mountscript.sh ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/scripts/aznfs_install.sh ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ + + mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${system_dir} + cp -avf ${SOURCE_DIR}/src/aznfswatchdog.service ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${system_dir} + cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4.service ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${system_dir} + + ########################################### + # Bundle aznfsclient and its dependencies # + ########################################### + + # copy the aznfsclient config file. + cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ + + # copy the aznfsclient binary. + cp -avf ${aznfsclient} ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient + + + if [ "$rpm_dir" != "azurelinux" ]; then + # + # Package aznfsclient dependencies in opt_dir/libs. + # libs_dir must already be populated with the required dependencies from + # the debian packaging step. Simply copy all those to rpm_libs_dir. + # + rpm_libs_dir=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/libs + mkdir -p ${rpm_libs_dir} + cp -avfH ${libs_dir}/* ${rpm_libs_dir} + fi + + # Create the archive for the package. + tar -cvzf ${STG_DIR}/${rpm_pkg_dir}.tar.gz -C ${STG_DIR}/${rpm_dir}/tmp root + + # Copy the SPEC file to change the placeholders depending upon the RPM distro. + cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/RPM/aznfs.spec ${STG_DIR}/${rpm_dir}/tmp/ + + if [ "$rpm_dir" != "azurelinux" ]; then + # + # Insert the contents of ${rpm_libs_dir}. + # This is variable due to the shared library versions. + # sed doesn't (easily) support replace target to be multi-line, so we use + # awk for this one. + # + opt_libs=$(for lib in ${rpm_libs_dir}/*; do echo ${opt_dir}/libs/$(basename $lib); done) + awk -i inplace -v r="$opt_libs" '{gsub(/OPT_LIBS/,r)}1' ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + fi + + # Insert current release number and RPM_DIR value. + sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + sed -i -e "s/RPM_DIR/${rpm_dir}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + sed -i -e "s/BUILD_ARCH/${arch}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + + # Replace the placeholders for various package names in aznfs.spec file. + if [ "$rpm_dir" == "suse" ]; then + sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}_sles/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + sed -i -e "s/NETCAT_PACKAGE_NAME/netcat-openbsd/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + # For SLES, sysvinit-tools provides pidof. + sed -i -e "s/PROCPS_PACKAGE_NAME/sysvinit-tools/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + sed -i -e "s/DISTRO/suse/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + else + if [ "$rpm_dir" == "stunnel" ]; then + sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}_stunnel_custom/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + else + sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + fi + + sed -i -e "s/NETCAT_PACKAGE_NAME/nmap-ncat/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + # In new versions of Centos/RedHat/Rocky, procps-ng provides pidof. For older versions, it is provided by sysvinit-tools but since it is not + # present in new versions, only install procps-ng which exists in all versions. + sed -i -e "s/PROCPS_PACKAGE_NAME/procps-ng/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + sed -i -e "s/DISTRO/rpm/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + sed -i -e "s/INSTALL_CMD/yum/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + fi + + # Create the rpm package. + rpmbuild --define "custom_stunnel $custom_stunnel_required" --define "azurelinux_build $azurelinux_build_required" --define "_topdir ${STG_DIR}/${rpm_dir}${rpmbuild_dir}" -v -bb ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec +} + +#STG_DIR, RELEASE_NUMBER and SOURCE_DIR will be taken as env var. +pkg_name="aznfs" +pkg_dir="${pkg_name}-${RELEASE_NUMBER}-1_$debarch" +rpm_pkg_dir="${pkg_name}-${RELEASE_NUMBER}-1.$arch" +opt_dir="/opt/microsoft/${pkg_name}" +system_dir="/lib/systemd/system" +rpmbuild_dir="/root/rpmbuild" +rpm_buildroot_dir="${rpmbuild_dir}/BUILDROOT" + +# Insert release number to aznfs_install.sh +sed -i -e "s/RELEASE_NUMBER=x.y.z/RELEASE_NUMBER=${RELEASE_NUMBER}/g" ${SOURCE_DIR}/scripts/aznfs_install.sh + +# +# We build the turbonfs project here, note that we can set all cmake options in the +# future using env variables. +# +pushd ${SOURCE_DIR}/turbonfs +export VCPKG_ROOT=${SOURCE_DIR}/turbonfs/extern/vcpkg +# We need to update the submodules before calling cmake as toolchain build expects it. +git submodule update --recursive --init +mkdir -p build && cd build + +if [ "${BUILD_TYPE}" == "Debug" ]; then + PARANOID=ON + INSECURE_AUTH_FOR_DEVTEST=ON +else + PARANOID=OFF + INSECURE_AUTH_FOR_DEVTEST=OFF +fi + +# Run azurelinux packaging only on azurelinux runner +if [ "$BUILD_MACHINE" == "azurelinux" ]; then + DYNAMIC_LINKS=ON +else + DYNAMIC_LINKS=OFF +fi + +# vcpkg required env variable VCPKG_FORCE_SYSTEM_BINARIES to be set for arm64. +if [ "$(uname -m)" == "aarch64" ]; then + export VCPKG_FORCE_SYSTEM_BINARIES=1 +fi + +cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + -DENABLE_PARANOID=${PARANOID} \ + -DENABLE_INSECURE_AUTH_FOR_DEVTEST=${INSECURE_AUTH_FOR_DEVTEST} \ + -DENABLE_DYNAMIC_LINKS=${DYNAMIC_LINKS} \ + -DPACKAGE_VERSION="${RELEASE_NUMBER}" \ + -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake .. +make +popd + +# Run azurelinux packaging only on azurelinux runner +if [ "$BUILD_MACHINE" == "azurelinux" ]; then + generate_rpm_package azurelinux +fi From d425299dbf742375ba795ecf4f6f94d3bd319757 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Tue, 23 Sep 2025 11:24:40 +0000 Subject: [PATCH 10/26] update --- package_azurelinux.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/package_azurelinux.sh b/package_azurelinux.sh index 86a6ebb1..036852e3 100644 --- a/package_azurelinux.sh +++ b/package_azurelinux.sh @@ -171,6 +171,7 @@ fi if [ "$(uname -m)" == "aarch64" ]; then export VCPKG_FORCE_SYSTEM_BINARIES=1 fi +export VCPKG_FORCE_SYSTEM_BINARIES=1 cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DENABLE_PARANOID=${PARANOID} \ From 8173c93d669c3d7eb1567c0af8cfe67c4414bf72 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Tue, 23 Sep 2025 14:20:39 +0000 Subject: [PATCH 11/26] update --- package_azurelinux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package_azurelinux.sh b/package_azurelinux.sh index 036852e3..b337544f 100644 --- a/package_azurelinux.sh +++ b/package_azurelinux.sh @@ -171,7 +171,7 @@ fi if [ "$(uname -m)" == "aarch64" ]; then export VCPKG_FORCE_SYSTEM_BINARIES=1 fi -export VCPKG_FORCE_SYSTEM_BINARIES=1 +# export VCPKG_FORCE_SYSTEM_BINARIES=1 cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DENABLE_PARANOID=${PARANOID} \ From 14169942e1d19fbe01ba70de9078358218c0c9b4 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Wed, 24 Sep 2025 05:47:56 +0000 Subject: [PATCH 12/26] update --- turbonfs/CMakeLists.txt | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index d01baca7..30fbf907 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -146,16 +146,23 @@ if(NOT azure-storage-blobs-cpp_FOUND) message(FATAL_ERROR "azure-storage-blobs-cpp not found! Did you run cmake with -DCMAKE_TOOLCHAIN_FILE?") endif() +# Choose package manager based on ENABLE_DYNAMIC_LINKS +if(ENABLE_DYNAMIC_LINKS) + set(PKG_MGR tdnf) +else() + set(PKG_MGR apt) +endif() + # # Ensure pkg_search_module(), needed for configuring some packages. # find_package(PkgConfig QUIET) if(NOT PKG_CONFIG_FOUND) message(STATUS "pkg-config not found, trying to install pkg-config!") - execute_process(COMMAND sudo apt install -y pkg-config + execute_process(COMMAND sudo ${PKG_MGR} install -y pkg-config RESULT_VARIABLE PKGCONFIG_INSTALL_RESULT) if(NOT PKGCONFIG_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install pkg-config failed with ${PKGCONFIG_INSTALL_RESULT}, try installing pkg-config manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install pkg-config failed with ${PKGCONFIG_INSTALL_RESULT}, try installing pkg-config manually and then run cmake again") else() # Call once more to ensure above install completed fine and also # add the pkg_search_module() command. @@ -169,10 +176,10 @@ endif() find_program(MESON_EXECUTABLE meson QUIET) if(NOT MESON_FOUND) message(STATUS "meson not found, trying to install meson!") - execute_process(COMMAND sudo apt install -y meson + execute_process(COMMAND sudo ${PKG_MGR} install -y meson RESULT_VARIABLE MESON_INSTALL_RESULT) if(NOT MESON_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install meson failed with ${MESON_INSTALL_RESULT}, try installing meson manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install meson failed with ${MESON_INSTALL_RESULT}, try installing meson manually and then run cmake again") else() find_program(MESON_EXECUTABLE meson REQUIRED) endif() @@ -184,10 +191,10 @@ endif() find_program(NINJA_EXECUTABLE ninja QUIET) if(NOT NINJA_FOUND) message(STATUS "ninja not found, trying to install ninja-build!") - execute_process(COMMAND sudo apt install -y ninja-build + execute_process(COMMAND sudo ${PKG_MGR} install -y ninja-build RESULT_VARIABLE NINJA_INSTALL_RESULT) if(NOT NINJA_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install ninja-build failed with ${NINJA_INSTALL_RESULT}, try installing ninja-build manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ninja-build failed with ${NINJA_INSTALL_RESULT}, try installing ninja-build manually and then run cmake again") else() find_program(NINJA_EXECUTABLE ninja REQUIRED) endif() @@ -212,10 +219,10 @@ add_custom_target(libfuse ALL DEPENDS ${fuse3_LIBRARY}) find_package(GnuTLS "3.4.6" QUIET) if(NOT GNUTLS_FOUND) message(STATUS "GnuTLS not found, trying to install gnutls-dev!") - execute_process(COMMAND sudo apt install -y gnutls-dev + execute_process(COMMAND sudo ${PKG_MGR} install -y gnutls-dev RESULT_VARIABLE GNUTLS_INSTALL_RESULT) if(NOT GNUTLS_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install gnutls-dev failed with ${GNUTLS_INSTALL_RESULT}, try installing gnutls-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install gnutls-dev failed with ${GNUTLS_INSTALL_RESULT}, try installing gnutls-dev manually and then run cmake again") else() find_package(GnuTLS "3.4.6" REQUIRED) endif() @@ -232,10 +239,10 @@ endif() find_package(tcmalloc QUIET) if(NOT tcmalloc_FOUND) message(STATUS "tcmalloc not found, trying to install libgoogle-perftools-dev!") - execute_process(COMMAND sudo apt install -y libgoogle-perftools-dev + execute_process(COMMAND sudo ${PKG_MGR} install -y libgoogle-perftools-dev RESULT_VARIABLE tcmalloc_INSTALL_RESULT) if(NOT tcmalloc_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install libgoogle-perftools-dev failed with ${tcmalloc_INSTALL_RESULT}, try installing libgoogle-perftools-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install libgoogle-perftools-dev failed with ${tcmalloc_INSTALL_RESULT}, try installing libgoogle-perftools-dev manually and then run cmake again") else() # Call once more to ensure above install completed fine and also # will set tcmalloc_INCLUDE_DIR and tcmalloc_LIBRARY variables. @@ -259,10 +266,10 @@ pkg_search_module(JEMALLOC QUIET jemalloc) if(NOT JEMALLOC_FOUND) message(STATUS "jemalloc not found, trying to install libjemalloc-dev!") - execute_process(COMMAND sudo apt install -y libjemalloc-dev + execute_process(COMMAND sudo ${PKG_MGR} install -y libjemalloc-dev RESULT_VARIABLE JEMALLOC_INSTALL_RESULT) if(NOT JEMALLOC_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install libjemalloc-dev failed with ${JEMALLOC_INSTALL_RESULT}, try installing libjemalloc-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install libjemalloc-dev failed with ${JEMALLOC_INSTALL_RESULT}, try installing libjemalloc-dev manually and then run cmake again") else() # Call once more to ensure above install completed fine and also # will set JEMALLOC_LIBRARIES variable. @@ -284,10 +291,10 @@ endif() pkg_search_module(UUID QUIET uuid) if(NOT UUID_FOUND) message(STATUS "uuid not found, trying to install uuid-dev!") - execute_process(COMMAND sudo apt install -y uuid-dev + execute_process(COMMAND sudo ${PKG_MGR} install -y uuid-dev RESULT_VARIABLE UUID_INSTALL_RESULT) if(NOT UUID_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install uuid-dev failed with ${UUID_INSTALL_RESULT}, try installing uuid-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install uuid-dev failed with ${UUID_INSTALL_RESULT}, try installing uuid-dev manually and then run cmake again") else() pkg_search_module(UUID REQUIRED uuid) endif() @@ -302,10 +309,10 @@ message(STATUS "Using uuid include dir ${UUID_INCLUDE_DIRS}") find_package(ZLIB QUIET) if(NOT ZLIB_FOUND) message(STATUS "zlib not found, trying to install zlib1g-dev!") - execute_process(COMMAND sudo apt install -y zlib1g-dev + execute_process(COMMAND sudo ${PKG_MGR} install -y zlib1g-dev RESULT_VARIABLE ZLIB_INSTALL_RESULT) if(NOT ZLIB_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install zlib1g-dev failed with ${ZLIB_INSTALL_RESULT}, try installing zlib1g-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install zlib1g-dev failed with ${ZLIB_INSTALL_RESULT}, try installing zlib1g-dev manually and then run cmake again") else() # Call once more to ensure above install completed fine and also # will set ZLIB_INCLUDE_DIR and ZLIB_LIBRARY variables. From ead8536f5de06a5b402df2c35b300e0dcf9e92c6 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Wed, 24 Sep 2025 05:53:40 +0000 Subject: [PATCH 13/26] update --- turbonfs/CMakeLists.txt | 87 +++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 42 deletions(-) diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index 30fbf907..f7661362 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -390,49 +390,52 @@ target_compile_options(${CMAKE_PROJECT_NAME} PRIVATE -Werror ) -# -# Libraries for statically linking gnutls. -# libp11-kit and libunistring do not have a static version available, so we are -# forced to use the shared version and bundle it. -# -set(CMAKE_FIND_LIBRARY_SUFFIXES .a) +# Only use static gnutls linking when ENABLE_DYNAMIC_LINKS is NOT set. +if(NOT ENABLE_DYNAMIC_LINKS) + # + # Libraries for statically linking gnutls. + # libp11-kit and libunistring do not have a static version available, so we are + # forced to use the shared version and bundle it. + # + set(CMAKE_FIND_LIBRARY_SUFFIXES .a) + + find_library(GNUTLS_STATIC_LIB libgnutls.a REQUIRED) + # + # arm64 libgnutls.a has multiple definitions for some symbols exported by + # libcrypto.a. The libcrypto.a object has some more symbols which are needed, + # so we delete the offending object from libgnutls.a before linking. + # + if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") + execute_process( + COMMAND bash -c "cp -vf ${GNUTLS_STATIC_LIB} /tmp/gnutls.a; ar dv /tmp/gnutls.a aes-aarch64.o" + COMMAND_ERROR_IS_FATAL ANY) + set(GNUTLS_STATIC_LIB /tmp/gnutls.a) + endif() -find_library(GNUTLS_STATIC_LIB libgnutls.a REQUIRED) -# -# arm64 libgnutls.a has multiple definitions for some symbols exported by -# libcrypto.a. The libcrypto.a object has some more symbols which are needed, -# so we delete the offending object from libgnutls.a before linking. -# -if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") -execute_process( - COMMAND bash -c "cp -vf ${GNUTLS_STATIC_LIB} /tmp/gnutls.a; ar dv /tmp/gnutls.a aes-aarch64.o" - COMMAND_ERROR_IS_FATAL ANY) -set(GNUTLS_STATIC_LIB /tmp/gnutls.a) -endif() - -find_library(HOGWEED_STATIC_LIB libhogweed.a REQUIRED) -find_library(NETTLE_STATIC_LIB libnettle.a REQUIRED) -find_library(TASN1_STATIC_LIB libtasn1.a REQUIRED) -find_library(IDN2_STATIC_LIB libidn2.a REQUIRED) -find_library(GMP_STATIC_LIB libgmp.a REQUIRED) -set(CMAKE_FIND_LIBRARY_SUFFIXES .so) -find_library(P11_KIT_SHARED_LIB libp11-kit.so REQUIRED) -find_library(UNISTRING_SHARED_LIB NAMES - libunistring.so - libunistring.so.5 - libunistring.so.2 - REQUIRED) - -set(GNUTLS_ALL_LIBRARIES - ${GNUTLS_STATIC_LIB} - ${HOGWEED_STATIC_LIB} - ${NETTLE_STATIC_LIB} - ${TASN1_STATIC_LIB} - ${IDN2_STATIC_LIB} - ${GMP_STATIC_LIB} - ${P11_KIT_SHARED_LIB} - ${UNISTRING_SHARED_LIB}) -message(STATUS "Using gnutls libraries: ${GNUTLS_ALL_LIBRARIES}") + find_library(HOGWEED_STATIC_LIB libhogweed.a REQUIRED) + find_library(NETTLE_STATIC_LIB libnettle.a REQUIRED) + find_library(TASN1_STATIC_LIB libtasn1.a REQUIRED) + find_library(IDN2_STATIC_LIB libidn2.a REQUIRED) + find_library(GMP_STATIC_LIB libgmp.a REQUIRED) + set(CMAKE_FIND_LIBRARY_SUFFIXES .so) + find_library(P11_KIT_SHARED_LIB libp11-kit.so REQUIRED) + find_library(UNISTRING_SHARED_LIB NAMES + libunistring.so + libunistring.so.5 + libunistring.so.2 + REQUIRED) + + set(GNUTLS_ALL_LIBRARIES + ${GNUTLS_STATIC_LIB} + ${HOGWEED_STATIC_LIB} + ${NETTLE_STATIC_LIB} + ${TASN1_STATIC_LIB} + ${IDN2_STATIC_LIB} + ${GMP_STATIC_LIB} + ${P11_KIT_SHARED_LIB} + ${UNISTRING_SHARED_LIB}) + message(STATUS "Using gnutls libraries: ${GNUTLS_ALL_LIBRARIES}") +endif() # All libraries. if(ENABLE_DYNAMIC_LINKS) From ca4e396809adb1a221d6ccdc21cfa37c14235033 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Wed, 24 Sep 2025 06:13:10 +0000 Subject: [PATCH 14/26] update --- turbonfs/CMakeLists.txt | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index f7661362..e3cd3c6d 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -200,10 +200,22 @@ if(NOT NINJA_FOUND) endif() endif() -# Meson will install the fuse library and header files in the following paths. -set(fuse3_LIBRARY "/usr/local/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/libfuse3.a") +# Decide fuse3 paths based on ENABLE_DYNAMIC_LINKS. +if(ENABLE_DYNAMIC_LINKS) + # Dynamic: Meson installs directly to /usr/local/lib. + set(fuse3_LIBRARY "/usr/local/lib/libfuse3.a") +else() + # Static: Use multiarch layout. + set(fuse3_LIBRARY "/usr/local/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/libfuse3.a") +endif() + +# Include path remains the same for both cases. set(fuse3_INCLUDE_DIR "/usr/local/include") +# Meson will install the fuse library and header files in the following paths. +# set(fuse3_LIBRARY "/usr/local/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/libfuse3.a") +# set(fuse3_INCLUDE_DIR "/usr/local/include") + add_custom_command( OUTPUT ${fuse3_LIBRARY} COMMAND ${MESON_EXECUTABLE} setup ${LIBFUSE_BUILD_DIR} ${LIBFUSE_SOURCE_DIR} --default-library=static --buildtype=${MESON_BUILD_TYPE} From a1033e87fce18b8535ad4ab5f323900df43a21ff Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Wed, 24 Sep 2025 06:31:47 +0000 Subject: [PATCH 15/26] update --- package_azurelinux.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/package_azurelinux.sh b/package_azurelinux.sh index b337544f..02e15462 100644 --- a/package_azurelinux.sh +++ b/package_azurelinux.sh @@ -68,9 +68,19 @@ generate_rpm_package() cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ # copy the aznfsclient binary. - cp -avf ${aznfsclient} ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient + # cp -avf ${aznfsclient} ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient + # Define the final target location for aznfsclient + aznfsclient_target="${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient" - + # Copy the built aznfsclient binary to the target + cp -avf "${SOURCE_DIR}/turbonfs/build/aznfsclient" "${aznfsclient_target}" + + # Optional: fail early if copy fails + if [ ! -f "${aznfsclient_target}" ]; then + echo "Error: aznfsclient failed to copy to ${aznfsclient_target}" + exit 1 + fi + if [ "$rpm_dir" != "azurelinux" ]; then # # Package aznfsclient dependencies in opt_dir/libs. From 2bb9aedd324ae84cb0cd724a2961f4a14629bf95 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Wed, 24 Sep 2025 07:01:09 +0000 Subject: [PATCH 16/26] update --- .github/workflows/prerelease.yml | 1 - package_azurelinux.sh | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 17e4f26e..986b3eca 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -100,7 +100,6 @@ jobs: target_commitish: ${{ github.sha }} files: | ${{ github.workspace }}/azurelinux/rpm/root/rpmbuild/RPMS/x86_64/aznfs-azurelinux-${{ github.event.inputs.versionName }}-1.x86_64.rpm - ${{ github.workspace }}/azurelinux/deb/aznfs-azurelinux-${{ github.event.inputs.versionName }}-1_amd64.deb body: | New Release ${{ github.event.inputs.versionName }} diff --git a/package_azurelinux.sh b/package_azurelinux.sh index 02e15462..03f72f29 100644 --- a/package_azurelinux.sh +++ b/package_azurelinux.sh @@ -124,6 +124,8 @@ generate_rpm_package() else if [ "$rpm_dir" == "stunnel" ]; then sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}_stunnel_custom/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + elif [ "$rpm_dir" == "azurelinux" ]; then + sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}-azurelinux/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec else sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec fi From d7d98a00cb11c5dd0f09048066783e483f1f4953 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Thu, 25 Sep 2025 06:12:34 +0000 Subject: [PATCH 17/26] update --- packaging/aznfs/RPM/aznfs.spec | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packaging/aznfs/RPM/aznfs.spec b/packaging/aznfs/RPM/aznfs.spec index 7f7a28a3..dc76c158 100644 --- a/packaging/aznfs/RPM/aznfs.spec +++ b/packaging/aznfs/RPM/aznfs.spec @@ -8,7 +8,7 @@ URL: https://github.com/Azure/AZNFS-mount/blob/main/README.md Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, net-tools, binutils, kernel-headers, openssl, openssl-devel, gcc, make, wget Recommends: build-essential -%elif 0%{?mariner_build} +%elif 0%{?azurelinux_build} Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, stunnel, net-tools, libuuid, gnutls, jemalloc, libasan %else @@ -45,7 +45,9 @@ tar -xzvf ${STG_DIR}/AZNFS_PACKAGE_NAME-${RELEASE_NUMBER}-1.BUILD_ARCH.tar.gz -C /opt/microsoft/aznfs/aznfs_install.sh /lib/systemd/system/aznfswatchdog.service /lib/systemd/system/aznfswatchdogv4.service +%if !0%{?azurelinux_build} OPT_LIBS +%endif /opt/microsoft/aznfs/sample-turbo-config.yaml /sbin/aznfsclient From f7a8d3ffcb82c38b88b9609404a43f6166a6fc86 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Thu, 25 Sep 2025 06:31:07 +0000 Subject: [PATCH 18/26] update --- .github/workflows/prerelease.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 986b3eca..581ee993 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -86,7 +86,7 @@ jobs: - name: Run Package.sh for azurelinux run: | export RELEASE_NUMBER=${{ github.event.inputs.versionName }} - export STG_DIR=$GITHUB_WORKSPACE/azurelinux + export STG_DIR=$GITHUB_WORKSPACE export SOURCE_DIR=$GITHUB_WORKSPACE/azurelinux export BUILD_TYPE=${{ github.events.inputs.buildType }} export BUILD_MACHINE=azurelinux From 05340800b03032f60ede1f441b214d9c82bc97f0 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Thu, 25 Sep 2025 06:50:56 +0000 Subject: [PATCH 19/26] update --- .github/workflows/prerelease.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 581ee993..aaa9b342 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -99,7 +99,7 @@ jobs: tag_name: ${{ github.event.inputs.versionName }} target_commitish: ${{ github.sha }} files: | - ${{ github.workspace }}/azurelinux/rpm/root/rpmbuild/RPMS/x86_64/aznfs-azurelinux-${{ github.event.inputs.versionName }}-1.x86_64.rpm + ${{ github.workspace }}/azurelinux/root/rpmbuild/RPMS/x86_64/aznfs-azurelinux-${{ github.event.inputs.versionName }}-1.x86_64.rpm body: | New Release ${{ github.event.inputs.versionName }} From 4e620a2833c14da89f17c463be0f084ec6f7a256 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Thu, 25 Sep 2025 07:19:23 +0000 Subject: [PATCH 20/26] update --- packaging/aznfs/RPM/aznfs.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/aznfs/RPM/aznfs.spec b/packaging/aznfs/RPM/aznfs.spec index dc76c158..11f086b3 100644 --- a/packaging/aznfs/RPM/aznfs.spec +++ b/packaging/aznfs/RPM/aznfs.spec @@ -9,7 +9,7 @@ Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, ipro Recommends: build-essential %elif 0%{?azurelinux_build} -Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, stunnel, net-tools, libuuid, gnutls, jemalloc, libasan +Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, stunnel, net-tools, gnutls, jemalloc, libasan %else Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, stunnel, net-tools From 4433cf32fa4715f4b2417392a0b1ad81cc7ec0e3 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Thu, 25 Sep 2025 10:39:08 +0000 Subject: [PATCH 21/26] update --- turbonfs/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index e3cd3c6d..1cc7f555 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -24,9 +24,10 @@ option(ENABLE_NON_AZURE_NFS "Enable support for general NFS servers" OFF) option(ENABLE_CHATTY "Enable super verbose logs" OFF) option(ENABLE_TCMALLOC "Use tcmalloc for malloc/free/new/delete" OFF) option(ENABLE_JEMALLOC "Use jemalloc for malloc/free/new/delete" ON) -option(ENABLE_INSECURE_AUTH_FOR_DEVTEST "Enable AZAUTH for non-TLS connections" OFF) +option(ENABLE_INSECURE_AUTH_FOR_DEVTEST "Enable AZAUTH for non-TLS connections" ON) option(ENABLE_DYNAMIC_LINKS "Use dynamic linking instead of bundling libs" OFF) +message(STATUS "ENABLE_INSECURE_AUTH_FOR_DEVTEST = ${ENABLE_INSECURE_AUTH_FOR_DEVTEST}") # # Builds that make it to customers need to be extra careful about any unnecessary # logging. Some warning logs we have in our code are to attract developer From f6a9e95f99d17ccdefffe9bb5d9e7632e87fc6b0 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Thu, 25 Sep 2025 11:30:17 +0000 Subject: [PATCH 22/26] update --- .github/workflows/prerelease.yml | 6 +++--- turbonfs/CMakeLists.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index aaa9b342..7b99746b 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -54,7 +54,7 @@ jobs: export RELEASE_NUMBER=${{ github.event.inputs.versionName }} export STG_DIR=$GITHUB_WORKSPACE/amd64 export SOURCE_DIR=$GITHUB_WORKSPACE/amd64 - export BUILD_TYPE=${{ github.events.inputs.buildType }} + export BUILD_TYPE=${{ github.event.inputs.buildType }} export BUILD_MACHINE=ubuntu chmod +x $SOURCE_DIR/package.sh $SOURCE_DIR/package.sh @@ -88,7 +88,7 @@ jobs: export RELEASE_NUMBER=${{ github.event.inputs.versionName }} export STG_DIR=$GITHUB_WORKSPACE export SOURCE_DIR=$GITHUB_WORKSPACE/azurelinux - export BUILD_TYPE=${{ github.events.inputs.buildType }} + export BUILD_TYPE=${{ github.event.inputs.buildType }} export BUILD_MACHINE=azurelinux chmod +x $SOURCE_DIR/package_azurelinux.sh $SOURCE_DIR/package_azurelinux.sh @@ -130,7 +130,7 @@ jobs: export RELEASE_NUMBER=${{ github.event.inputs.versionName }} export STG_DIR=$GITHUB_WORKSPACE/arm64 export SOURCE_DIR=$GITHUB_WORKSPACE/arm64 - export BUILD_TYPE=${{ github.events.inputs.buildType }} + export BUILD_TYPE=${{ github.event.inputs.buildType }} chmod +x $SOURCE_DIR/package.sh $SOURCE_DIR/package.sh - name: Create Test Release diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index 1cc7f555..d35289ec 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -24,7 +24,7 @@ option(ENABLE_NON_AZURE_NFS "Enable support for general NFS servers" OFF) option(ENABLE_CHATTY "Enable super verbose logs" OFF) option(ENABLE_TCMALLOC "Use tcmalloc for malloc/free/new/delete" OFF) option(ENABLE_JEMALLOC "Use jemalloc for malloc/free/new/delete" ON) -option(ENABLE_INSECURE_AUTH_FOR_DEVTEST "Enable AZAUTH for non-TLS connections" ON) +option(ENABLE_INSECURE_AUTH_FOR_DEVTEST "Enable AZAUTH for non-TLS connections" OFF) option(ENABLE_DYNAMIC_LINKS "Use dynamic linking instead of bundling libs" OFF) message(STATUS "ENABLE_INSECURE_AUTH_FOR_DEVTEST = ${ENABLE_INSECURE_AUTH_FOR_DEVTEST}") From 55a2689a394d010ee72fe540a581cbe0e93d928d Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Fri, 26 Sep 2025 05:50:01 +0000 Subject: [PATCH 23/26] revert to original package.sh --- package.sh | 108 +++++++++++++++++++---------------------------------- 1 file changed, 38 insertions(+), 70 deletions(-) diff --git a/package.sh b/package.sh index 9c896e2c..053426d8 100755 --- a/package.sh +++ b/package.sh @@ -24,19 +24,12 @@ generate_rpm_package() { rpm_dir=$1 custom_stunnel_required=0 - azurelinux_build_required=0 # Overwrite rpm_pkg_dir in case of SUSE. if [ "$rpm_dir" == "suse" ]; then rpm_pkg_dir="${pkg_name}_sles-${RELEASE_NUMBER}-1.$arch" fi - # Overwrite rpm_pkg_dir in case of azurelinux. - if [ "$rpm_dir" == "azurelinux" ]; then - rpm_pkg_dir="${pkg_name}-azurelinux-${RELEASE_NUMBER}-1.$arch" - azurelinux_build_required=1 - fi - # Overwrite rpm_pkg_dir in case of Mariner, RedHat7, and Centos7. if [ "$rpm_dir" == "stunnel" ]; then rpm_pkg_dir="${pkg_name}_stunnel_custom-${RELEASE_NUMBER}-1.$arch" @@ -76,17 +69,14 @@ generate_rpm_package() # copy the aznfsclient binary. cp -avf ${aznfsclient} ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient - - if [ "$rpm_dir" != "azurelinux" ]; then - # - # Package aznfsclient dependencies in opt_dir/libs. - # libs_dir must already be populated with the required dependencies from - # the debian packaging step. Simply copy all those to rpm_libs_dir. - # - rpm_libs_dir=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/libs - mkdir -p ${rpm_libs_dir} - cp -avfH ${libs_dir}/* ${rpm_libs_dir} - fi + # + # Package aznfsclient dependencies in opt_dir/libs. + # libs_dir must already be populated with the required dependencies from + # the debian packaging step. Simply copy all those to rpm_libs_dir. + # + rpm_libs_dir=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/libs + mkdir -p ${rpm_libs_dir} + cp -avfH ${libs_dir}/* ${rpm_libs_dir} # Create the archive for the package. tar -cvzf ${STG_DIR}/${rpm_pkg_dir}.tar.gz -C ${STG_DIR}/${rpm_dir}/tmp root @@ -94,16 +84,14 @@ generate_rpm_package() # Copy the SPEC file to change the placeholders depending upon the RPM distro. cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/RPM/aznfs.spec ${STG_DIR}/${rpm_dir}/tmp/ - if [ "$rpm_dir" != "azurelinux" ]; then - # - # Insert the contents of ${rpm_libs_dir}. - # This is variable due to the shared library versions. - # sed doesn't (easily) support replace target to be multi-line, so we use - # awk for this one. - # - opt_libs=$(for lib in ${rpm_libs_dir}/*; do echo ${opt_dir}/libs/$(basename $lib); done) - awk -i inplace -v r="$opt_libs" '{gsub(/OPT_LIBS/,r)}1' ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec - fi + # + # Insert the contents of ${rpm_libs_dir}. + # This is variable due to the shared library versions. + # sed doesn't (easily) support replace target to be multi-line, so we use + # awk for this one. + # + opt_libs=$(for lib in ${rpm_libs_dir}/*; do echo ${opt_dir}/libs/$(basename $lib); done) + awk -i inplace -v r="$opt_libs" '{gsub(/OPT_LIBS/,r)}1' ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec # Insert current release number and RPM_DIR value. sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec @@ -133,7 +121,7 @@ generate_rpm_package() fi # Create the rpm package. - rpmbuild --define "custom_stunnel $custom_stunnel_required" --define "azurelinux_build $azurelinux_build_required" --define "_topdir ${STG_DIR}/${rpm_dir}${rpmbuild_dir}" -v -bb ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + rpmbuild --define "custom_stunnel $custom_stunnel_required" --define "_topdir ${STG_DIR}/${rpm_dir}${rpmbuild_dir}" -v -bb ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec } generate_tarball_package() @@ -237,12 +225,7 @@ cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ # Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin -if [ "$BUILD_MACHINE" == "azurelinux" ]; then -gcc ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs -else gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs -fi -# gcc ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs # # We build the turbonfs project here, note that we can set all cmake options in the @@ -263,13 +246,6 @@ else INSECURE_AUTH_FOR_DEVTEST=OFF fi -# Run azurelinux packaging only on azurelinux runner -if [ "$BUILD_MACHINE" == "azurelinux" ]; then - DYNAMIC_LINKS=ON -else - DYNAMIC_LINKS=OFF -fi - # vcpkg required env variable VCPKG_FORCE_SYSTEM_BINARIES to be set for arm64. if [ "$(uname -m)" == "aarch64" ]; then export VCPKG_FORCE_SYSTEM_BINARIES=1 @@ -278,7 +254,6 @@ fi cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DENABLE_PARANOID=${PARANOID} \ -DENABLE_INSECURE_AUTH_FOR_DEVTEST=${INSECURE_AUTH_FOR_DEVTEST} \ - -DENABLE_DYNAMIC_LINKS=${DYNAMIC_LINKS} \ -DPACKAGE_VERSION="${RELEASE_NUMBER}" \ -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake .. make @@ -300,38 +275,36 @@ cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4.service ${STG_DIR}/deb/${pkg_dir}${sys # Bundle aznfsclient and its dependencies # ########################################### -# # aznfsclient in the final target dir. +# aznfsclient in the final target dir. aznfsclient=${STG_DIR}/deb/${pkg_dir}/sbin/aznfsclient cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} -# # Package aznfsclient dependencies in opt_dir. +# Package aznfsclient dependencies in opt_dir. libs_dir=${STG_DIR}/deb/${pkg_dir}${opt_dir}/libs mkdir -p ${libs_dir} -# # Copy the dependencies. +# Copy the dependencies. cp -avfH $(ldd ${aznfsclient} | grep "=>" | awk '{print $3}') ${libs_dir} -if [ "$BUILD_MACHINE" != "azurelinux" ]; then - # - # Patch all the libs to reference shared libs from ${libs_dir}. - # This is our very simple containerization. - # - for lib in ${libs_dir}/*.so*; do - echo "Setting rpath to ${opt_dir}/libs for $lib" - patchelf --set-rpath ${opt_dir}/libs "$lib" - done +# +# Patch all the libs to reference shared libs from ${libs_dir}. +# This is our very simple containerization. +# +for lib in ${libs_dir}/*.so*; do + echo "Setting rpath to ${opt_dir}/libs for $lib" + patchelf --set-rpath ${opt_dir}/libs "$lib" +done - # - # Final containerization effort - bundle and use the same interpreter as the - # build machine. - # - ld_linux_path=$(ldd ${aznfsclient} | grep "ld-linux" | awk '{print $1}') - ld_linux_name=$(basename "$ld_linux_path") - ld_linux="${libs_dir}/${ld_linux_name}" - cp -avfH "${ld_linux_path}" "${ld_linux}" +# +# Final containerization effort - bundle and use the same interpreter as the +# build machine. +# +ld_linux_path=$(ldd ${aznfsclient} | grep "ld-linux" | awk '{print $1}') +ld_linux_name=$(basename "$ld_linux_path") +ld_linux="${libs_dir}/${ld_linux_name}" +cp -avfH "${ld_linux_path}" "${ld_linux}" - patchelf --set-interpreter ${opt_dir}/libs/${ld_linux_name} ${aznfsclient} -fi +patchelf --set-interpreter ${opt_dir}/libs/${ld_linux_name} ${aznfsclient} # Create the deb package. dpkg-deb -Zgzip --root-owner-group --build $STG_DIR/deb/$pkg_dir @@ -345,13 +318,8 @@ generate_rpm_package suse # Generate rpm package with custom stunnel installation for Mariner, RedHat7, and Centos7. generate_rpm_package stunnel -# Run azurelinux packaging only on azurelinux runner -if [ "$BUILD_MACHINE" == "azurelinux" ]; then - generate_rpm_package azurelinux -fi - ############################# # Generating Tarball for AKS# ############################# -generate_tarball_package $arch +generate_tarball_package $arch \ No newline at end of file From affea5fccec05af5d19998b1dae53be9aee58a92 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Fri, 26 Sep 2025 06:55:06 +0000 Subject: [PATCH 24/26] update for package.sh including all changes --- .github/workflows/prerelease.yml | 4 +- package.sh | 219 ++++++++++++++++++------------- 2 files changed, 132 insertions(+), 91 deletions(-) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 7b99746b..a231e3e6 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -90,8 +90,8 @@ jobs: export SOURCE_DIR=$GITHUB_WORKSPACE/azurelinux export BUILD_TYPE=${{ github.event.inputs.buildType }} export BUILD_MACHINE=azurelinux - chmod +x $SOURCE_DIR/package_azurelinux.sh - $SOURCE_DIR/package_azurelinux.sh + chmod +x $SOURCE_DIR/package.sh + $SOURCE_DIR/package.sh - name: Create Release (azurelinux) uses: softprops/action-gh-release@v1 with: diff --git a/package.sh b/package.sh index 053426d8..44cf8246 100755 --- a/package.sh +++ b/package.sh @@ -24,12 +24,19 @@ generate_rpm_package() { rpm_dir=$1 custom_stunnel_required=0 + azurelinux_build_required=0 # Overwrite rpm_pkg_dir in case of SUSE. if [ "$rpm_dir" == "suse" ]; then rpm_pkg_dir="${pkg_name}_sles-${RELEASE_NUMBER}-1.$arch" fi + # Overwrite rpm_pkg_dir in case of azurelinux. + if [ "$rpm_dir" == "azurelinux" ]; then + rpm_pkg_dir="${pkg_name}-azurelinux-${RELEASE_NUMBER}-1.$arch" + azurelinux_build_required=1 + fi + # Overwrite rpm_pkg_dir in case of Mariner, RedHat7, and Centos7. if [ "$rpm_dir" == "stunnel" ]; then rpm_pkg_dir="${pkg_name}_stunnel_custom-${RELEASE_NUMBER}-1.$arch" @@ -66,17 +73,33 @@ generate_rpm_package() # copy the aznfsclient config file. cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ - # copy the aznfsclient binary. - cp -avf ${aznfsclient} ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient + if [ "$rpm_dir" != "azurelinux" ]; then + # copy the aznfsclient binary. + cp -avf ${aznfsclient} ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient + else + # Define the final target location for aznfsclient + aznfsclient_target="${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient" - # - # Package aznfsclient dependencies in opt_dir/libs. - # libs_dir must already be populated with the required dependencies from - # the debian packaging step. Simply copy all those to rpm_libs_dir. - # - rpm_libs_dir=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/libs - mkdir -p ${rpm_libs_dir} - cp -avfH ${libs_dir}/* ${rpm_libs_dir} + # Copy the built aznfsclient binary to the target + cp -avf "${SOURCE_DIR}/turbonfs/build/aznfsclient" "${aznfsclient_target}" + + # Optional: fail early if copy fails + if [ ! -f "${aznfsclient_target}" ]; then + echo "Error: aznfsclient failed to copy to ${aznfsclient_target}" + exit 1 + fi + fi + + if [ "$rpm_dir" != "azurelinux" ]; then + # + # Package aznfsclient dependencies in opt_dir/libs. + # libs_dir must already be populated with the required dependencies from + # the debian packaging step. Simply copy all those to rpm_libs_dir. + # + rpm_libs_dir=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/libs + mkdir -p ${rpm_libs_dir} + cp -avfH ${libs_dir}/* ${rpm_libs_dir} + fi # Create the archive for the package. tar -cvzf ${STG_DIR}/${rpm_pkg_dir}.tar.gz -C ${STG_DIR}/${rpm_dir}/tmp root @@ -84,20 +107,22 @@ generate_rpm_package() # Copy the SPEC file to change the placeholders depending upon the RPM distro. cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/RPM/aznfs.spec ${STG_DIR}/${rpm_dir}/tmp/ - # - # Insert the contents of ${rpm_libs_dir}. - # This is variable due to the shared library versions. - # sed doesn't (easily) support replace target to be multi-line, so we use - # awk for this one. - # - opt_libs=$(for lib in ${rpm_libs_dir}/*; do echo ${opt_dir}/libs/$(basename $lib); done) - awk -i inplace -v r="$opt_libs" '{gsub(/OPT_LIBS/,r)}1' ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + if [ "$rpm_dir" != "azurelinux" ]; then + # + # Insert the contents of ${rpm_libs_dir}. + # This is variable due to the shared library versions. + # sed doesn't (easily) support replace target to be multi-line, so we use + # awk for this one. + # + opt_libs=$(for lib in ${rpm_libs_dir}/*; do echo ${opt_dir}/libs/$(basename $lib); done) + awk -i inplace -v r="$opt_libs" '{gsub(/OPT_LIBS/,r)}1' ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + fi # Insert current release number and RPM_DIR value. sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec sed -i -e "s/RPM_DIR/${rpm_dir}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec sed -i -e "s/BUILD_ARCH/${arch}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec - + # Replace the placeholders for various package names in aznfs.spec file. if [ "$rpm_dir" == "suse" ]; then sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}_sles/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec @@ -108,6 +133,8 @@ generate_rpm_package() else if [ "$rpm_dir" == "stunnel" ]; then sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}_stunnel_custom/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + elif [ "$rpm_dir" == "azurelinux" ]; then + sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}-azurelinux/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec else sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec fi @@ -121,7 +148,7 @@ generate_rpm_package() fi # Create the rpm package. - rpmbuild --define "custom_stunnel $custom_stunnel_required" --define "_topdir ${STG_DIR}/${rpm_dir}${rpmbuild_dir}" -v -bb ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + rpmbuild --define "custom_stunnel $custom_stunnel_required" --define "azurelinux_build $azurelinux_build_required" --define "_topdir ${STG_DIR}/${rpm_dir}${rpmbuild_dir}" -v -bb ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec } generate_tarball_package() @@ -203,29 +230,31 @@ rpm_buildroot_dir="${rpmbuild_dir}/BUILDROOT" # Insert release number to aznfs_install.sh sed -i -e "s/RELEASE_NUMBER=x.y.z/RELEASE_NUMBER=${RELEASE_NUMBER}/g" ${SOURCE_DIR}/scripts/aznfs_install.sh -######################### -# Generate .deb package # -######################### +if [ "$BUILD_MACHINE" != "azurelinux" ]; then + ######################### + # Generate .deb package # + ######################### -# Create the directory to hold the package control and data files for deb package. -mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN + # Create the directory to hold the package control and data files for deb package. + mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN -# Copy the debian control file(s) and maintainer scripts. -cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ -chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* + # Copy the debian control file(s) and maintainer scripts. + cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ + chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* -# Insert current release number. -sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control -sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control + # Insert current release number. + sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control + sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control -# Copy other static package file(s). -mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin -cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ -cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ + # Copy other static package file(s). + mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin + cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ + cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ -# Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. -mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin -gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs + # Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. + mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin + gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs +fi # # We build the turbonfs project here, note that we can set all cmake options in the @@ -246,6 +275,13 @@ else INSECURE_AUTH_FOR_DEVTEST=OFF fi +# Run azurelinux packaging only on azurelinux runner +if [ "$BUILD_MACHINE" == "azurelinux" ]; then + DYNAMIC_LINKS=ON +else + DYNAMIC_LINKS=OFF +fi + # vcpkg required env variable VCPKG_FORCE_SYSTEM_BINARIES to be set for arm64. if [ "$(uname -m)" == "aarch64" ]; then export VCPKG_FORCE_SYSTEM_BINARIES=1 @@ -254,72 +290,77 @@ fi cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DENABLE_PARANOID=${PARANOID} \ -DENABLE_INSECURE_AUTH_FOR_DEVTEST=${INSECURE_AUTH_FOR_DEVTEST} \ + -DENABLE_DYNAMIC_LINKS=${DYNAMIC_LINKS} \ -DPACKAGE_VERSION="${RELEASE_NUMBER}" \ -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake .. make popd -mkdir -p ${STG_DIR}/deb/${pkg_dir}${opt_dir} -cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/src/mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/src/nfsv3mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/src/nfsv4mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/scripts/aznfs_install.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/deb/${pkg_dir}/${opt_dir}/ +if [ "$BUILD_MACHINE" != "azurelinux" ]; then + mkdir -p ${STG_DIR}/deb/${pkg_dir}${opt_dir} + cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/nfsv3mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/nfsv4mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/scripts/aznfs_install.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/deb/${pkg_dir}/${opt_dir}/ -mkdir -p ${STG_DIR}/deb/${pkg_dir}${system_dir} -cp -avf ${SOURCE_DIR}/src/aznfswatchdog.service ${STG_DIR}/deb/${pkg_dir}${system_dir} -cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4.service ${STG_DIR}/deb/${pkg_dir}${system_dir} + mkdir -p ${STG_DIR}/deb/${pkg_dir}${system_dir} + cp -avf ${SOURCE_DIR}/src/aznfswatchdog.service ${STG_DIR}/deb/${pkg_dir}${system_dir} + cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4.service ${STG_DIR}/deb/${pkg_dir}${system_dir} -########################################### -# Bundle aznfsclient and its dependencies # -########################################### + ########################################### + # Bundle aznfsclient and its dependencies # + ########################################### -# aznfsclient in the final target dir. -aznfsclient=${STG_DIR}/deb/${pkg_dir}/sbin/aznfsclient -cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} + # aznfsclient in the final target dir. + aznfsclient=${STG_DIR}/deb/${pkg_dir}/sbin/aznfsclient + cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} -# Package aznfsclient dependencies in opt_dir. -libs_dir=${STG_DIR}/deb/${pkg_dir}${opt_dir}/libs -mkdir -p ${libs_dir} + # Package aznfsclient dependencies in opt_dir. + libs_dir=${STG_DIR}/deb/${pkg_dir}${opt_dir}/libs + mkdir -p ${libs_dir} -# Copy the dependencies. -cp -avfH $(ldd ${aznfsclient} | grep "=>" | awk '{print $3}') ${libs_dir} + # Copy the dependencies. + cp -avfH $(ldd ${aznfsclient} | grep "=>" | awk '{print $3}') ${libs_dir} -# -# Patch all the libs to reference shared libs from ${libs_dir}. -# This is our very simple containerization. -# -for lib in ${libs_dir}/*.so*; do - echo "Setting rpath to ${opt_dir}/libs for $lib" - patchelf --set-rpath ${opt_dir}/libs "$lib" -done + # + # Patch all the libs to reference shared libs from ${libs_dir}. + # This is our very simple containerization. + # + for lib in ${libs_dir}/*.so*; do + echo "Setting rpath to ${opt_dir}/libs for $lib" + patchelf --set-rpath ${opt_dir}/libs "$lib" + done -# -# Final containerization effort - bundle and use the same interpreter as the -# build machine. -# -ld_linux_path=$(ldd ${aznfsclient} | grep "ld-linux" | awk '{print $1}') -ld_linux_name=$(basename "$ld_linux_path") -ld_linux="${libs_dir}/${ld_linux_name}" -cp -avfH "${ld_linux_path}" "${ld_linux}" + # + # Final containerization effort - bundle and use the same interpreter as the + # build machine. + # + ld_linux_path=$(ldd ${aznfsclient} | grep "ld-linux" | awk '{print $1}') + ld_linux_name=$(basename "$ld_linux_path") + ld_linux="${libs_dir}/${ld_linux_name}" + cp -avfH "${ld_linux_path}" "${ld_linux}" -patchelf --set-interpreter ${opt_dir}/libs/${ld_linux_name} ${aznfsclient} + patchelf --set-interpreter ${opt_dir}/libs/${ld_linux_name} ${aznfsclient} -# Create the deb package. -dpkg-deb -Zgzip --root-owner-group --build $STG_DIR/deb/$pkg_dir + # Create the deb package. + dpkg-deb -Zgzip --root-owner-group --build $STG_DIR/deb/$pkg_dir -######################### -# Generate .rpm package # -######################### + ######################### + # Generate .rpm package # + ######################### -generate_rpm_package rpm -generate_rpm_package suse -# Generate rpm package with custom stunnel installation for Mariner, RedHat7, and Centos7. -generate_rpm_package stunnel + generate_rpm_package rpm + generate_rpm_package suse + # Generate rpm package with custom stunnel installation for Mariner, RedHat7, and Centos7. + generate_rpm_package stunnel -############################# -# Generating Tarball for AKS# -############################# + ############################# + # Generating Tarball for AKS# + ############################# -generate_tarball_package $arch \ No newline at end of file + generate_tarball_package $arch +else + generate_rpm_package azurelinux +fi \ No newline at end of file From 7dd767c288edbc0b87ee0bcfee00c108c375c9a8 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Fri, 26 Sep 2025 10:14:50 +0000 Subject: [PATCH 25/26] update --- .github/workflows/prerelease.yml | 12 ++++++++ .github/workflows/release.yml | 30 ------------------ turbonfs/CMakeLists.txt | 53 +++++++++++++++++++++++--------- 3 files changed, 51 insertions(+), 44 deletions(-) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index a231e3e6..ce0508c9 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -83,6 +83,17 @@ jobs: uses: actions/checkout@v3 with: path: azurelinux + + - name: Install prerequisites on Azure Linux + run: | + sudo tdnf update -y + sudo tdnf install -y glibc-static rpm-build perl perl-IPC-Cmd \ + gcc gcc-c++ make git curl unzip zip tar cmake \ + zlib zlib-devel libuuid-devel gnutls gnutls-devel nfs-utils \ + libyaml-devel spdlog-devel nlohmann-json-devel cmake make \ + pkgconf pkgconf-m4 pkgconf-pkg-config ninja-build binutils \ + kernel-headers kernel-devel autoconf automake libtool + - name: Run Package.sh for azurelinux run: | export RELEASE_NUMBER=${{ github.event.inputs.versionName }} @@ -92,6 +103,7 @@ jobs: export BUILD_MACHINE=azurelinux chmod +x $SOURCE_DIR/package.sh $SOURCE_DIR/package.sh + - name: Create Release (azurelinux) uses: softprops/action-gh-release@v1 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4021f56c..c6c81ce1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -93,36 +93,6 @@ jobs: body: | New Release ${{ github.event.inputs.versionName }} - package_mount_helper_azurelinux: - name: Package and Release ${{ github.event.inputs.versionName }} for azurelinux - runs-on: AzureLinux-3.0 - needs: verify_branch_name - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - path: azurelinux - - name: Run Package.sh for azurelinux - run: | - export RELEASE_NUMBER=${{ github.event.inputs.versionName }} - export STG_DIR=$GITHUB_WORKSPACE/azurelinux - export SOURCE_DIR=$GITHUB_WORKSPACE/azurelinux - export BUILD_TYPE=${{ github.events.inputs.buildType }} - export BUILD_MACHINE=azurelinux - chmod +x $SOURCE_DIR/package.sh - $SOURCE_DIR/package.sh - - name: Create Release (azurelinux) - uses: softprops/action-gh-release@v1 - with: - name: Release ${{ github.event.inputs.versionName }} - tag_name: ${{ github.event.inputs.versionName }} - target_commitish: ${{ github.sha }} - files: | - ${{ github.workspace }}/azurelinux/rpm/root/rpmbuild/RPMS/x86_64/aznfs-azurelinux-${{ github.event.inputs.versionName }}-1.x86_64.rpm - ${{ github.workspace }}/azurelinux/deb/aznfs-azurelinux-${{ github.event.inputs.versionName }}-1_amd64.deb - body: | - New Release ${{ github.event.inputs.versionName }} - package_mount_helper_arm64: name: Package and Release ${{ github.event.inputs.versionName }} for arm64 runs-on: ubuntu-22.04-arm diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index d35289ec..ab94a682 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -232,10 +232,15 @@ add_custom_target(libfuse ALL DEPENDS ${fuse3_LIBRARY}) find_package(GnuTLS "3.4.6" QUIET) if(NOT GNUTLS_FOUND) message(STATUS "GnuTLS not found, trying to install gnutls-dev!") - execute_process(COMMAND sudo ${PKG_MGR} install -y gnutls-dev + if(PKG_MGR STREQUAL "apt") + set(GNUTLS_PKG "gnutls-dev") + else() + set(GNUTLS_PKG "gnutls-devel") + endif() + execute_process(COMMAND sudo ${PKG_MGR} install -y ${GNUTLS_PKG} RESULT_VARIABLE GNUTLS_INSTALL_RESULT) if(NOT GNUTLS_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "${PKG_MGR} install gnutls-dev failed with ${GNUTLS_INSTALL_RESULT}, try installing gnutls-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${GNUTLS_PKG} failed with ${GNUTLS_INSTALL_RESULT}, try installing ${GNUTLS_PKG} manually and then run cmake again") else() find_package(GnuTLS "3.4.6" REQUIRED) endif() @@ -251,11 +256,16 @@ endif() find_package(tcmalloc QUIET) if(NOT tcmalloc_FOUND) - message(STATUS "tcmalloc not found, trying to install libgoogle-perftools-dev!") - execute_process(COMMAND sudo ${PKG_MGR} install -y libgoogle-perftools-dev + if(PKG_MGR STREQUAL "apt") + set(TCMALLOC_PKG "libgoogle-perftools-dev") + else() + set(TCMALLOC_PKG "gperftools-devel") + endif() + message(STATUS "tcmalloc not found, trying to install ${TCMALLOC_PKG}!") + execute_process(COMMAND sudo ${PKG_MGR} install -y ${TCMALLOC_PKG} RESULT_VARIABLE tcmalloc_INSTALL_RESULT) if(NOT tcmalloc_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "${PKG_MGR} install libgoogle-perftools-dev failed with ${tcmalloc_INSTALL_RESULT}, try installing libgoogle-perftools-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${TCMALLOC_PKG} failed with ${tcmalloc_INSTALL_RESULT}, try installing ${TCMALLOC_PKG} manually and then run cmake again") else() # Call once more to ensure above install completed fine and also # will set tcmalloc_INCLUDE_DIR and tcmalloc_LIBRARY variables. @@ -278,11 +288,16 @@ endif() pkg_search_module(JEMALLOC QUIET jemalloc) if(NOT JEMALLOC_FOUND) - message(STATUS "jemalloc not found, trying to install libjemalloc-dev!") - execute_process(COMMAND sudo ${PKG_MGR} install -y libjemalloc-dev + if(${PKG_MGR} STREQUAL "tdnf") + set(JEMALLOC_PKG jemalloc-devel) + else() + set(JEMALLOC_PKG libjemalloc-dev) + endif() + message(STATUS "jemalloc not found, trying to install ${JEMALLOC_PKG}!") + execute_process(COMMAND sudo ${PKG_MGR} install -y ${JEMALLOC_PKG} RESULT_VARIABLE JEMALLOC_INSTALL_RESULT) if(NOT JEMALLOC_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "${PKG_MGR} install libjemalloc-dev failed with ${JEMALLOC_INSTALL_RESULT}, try installing libjemalloc-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${JEMALLOC_PKG} failed with ${JEMALLOC_INSTALL_RESULT}, try installing ${JEMALLOC_PKG} manually and then run cmake again") else() # Call once more to ensure above install completed fine and also # will set JEMALLOC_LIBRARIES variable. @@ -303,11 +318,16 @@ endif() # pkg_search_module(UUID QUIET uuid) if(NOT UUID_FOUND) - message(STATUS "uuid not found, trying to install uuid-dev!") - execute_process(COMMAND sudo ${PKG_MGR} install -y uuid-dev + if(PKG_MGR STREQUAL "apt") + set(UUID_PKG "uuid-dev") + else() + set(UUID_PKG "uuid-devel") + endif() + message(STATUS "uuid not found, trying to install ${UUID_PKG}!") + execute_process(COMMAND sudo ${PKG_MGR} install -y ${UUID_PKG} RESULT_VARIABLE UUID_INSTALL_RESULT) if(NOT UUID_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "${PKG_MGR} install uuid-dev failed with ${UUID_INSTALL_RESULT}, try installing uuid-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${UUID_PKG} failed with ${UUID_INSTALL_RESULT}, try installing ${UUID_PKG} manually and then run cmake again") else() pkg_search_module(UUID REQUIRED uuid) endif() @@ -321,11 +341,16 @@ message(STATUS "Using uuid include dir ${UUID_INCLUDE_DIRS}") # find_package(ZLIB QUIET) if(NOT ZLIB_FOUND) - message(STATUS "zlib not found, trying to install zlib1g-dev!") - execute_process(COMMAND sudo ${PKG_MGR} install -y zlib1g-dev + if(PKG_MGR STREQUAL "apt") + set(ZLIB_PKG "zlib1g-dev") + else() + set(ZLIB_PKG "zlib-devel") + endif() + message(STATUS "zlib not found, trying to install ${ZLIB_PKG}!") + execute_process(COMMAND sudo ${PKG_MGR} install -y ${ZLIB_PKG} RESULT_VARIABLE ZLIB_INSTALL_RESULT) if(NOT ZLIB_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "${PKG_MGR} install zlib1g-dev failed with ${ZLIB_INSTALL_RESULT}, try installing zlib1g-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${ZLIB_PKG} failed with ${ZLIB_INSTALL_RESULT}, try installing ${ZLIB_PKG} manually and then run cmake again") else() # Call once more to ensure above install completed fine and also # will set ZLIB_INCLUDE_DIR and ZLIB_LIBRARY variables. From 26ed607732e64ba66853e12fc9874de97a43526c Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Mon, 29 Sep 2025 11:18:34 +0000 Subject: [PATCH 26/26] update main pipeline --- .github/aznfs-build.yaml | 60 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/.github/aznfs-build.yaml b/.github/aznfs-build.yaml index 22ad311b..71fd3eb8 100644 --- a/.github/aznfs-build.yaml +++ b/.github/aznfs-build.yaml @@ -78,6 +78,66 @@ stages: PathtoPublish: $(Build.ArtifactStagingDirectory) artifactName: 'aznfs-temp' + - job: package_aznfs_azurelinu_amd64 + displayName: Package and Release ${{ parameters.versionName }} for azurelinux + pool: + name: 'aznfsazurelinuxdemopool' # Use your self-hosted agent + + steps: + - checkout: self + path: azurelinux/AZNFS-mount + displayName: 'Checkout repository' + + - script: | + echo "Installing prerequisites on Azure Linux..." + sudo tdnf update -y + sudo tdnf install -y glibc-static rpm-build perl perl-IPC-Cmd \ + gcc gcc-c++ make git curl unzip zip tar cmake \ + zlib zlib-devel libuuid-devel gnutls gnutls-devel nfs-utils \ + libyaml-devel spdlog-devel nlohmann-json-devel cmake make \ + pkgconf pkgconf-m4 pkgconf-pkg-config ninja-build binutils \ + kernel-headers kernel-devel autoconf automake libtool jemalloc-devel + displayName: "Install prerequisites on Azure Linux" + + # note: remove later. + - script: | + echo "=== Validating Build Environment ===" + gcc --version + cmake --version + tdnf --version + echo "Architecture: $(uname -m)" + displayName: "Validate build environment" + + - script: | + export RELEASE_NUMBER=${{ parameters.versionName }} + export STG_DIR=$(System.DefaultWorkingDirectory) + export SOURCE_DIR=$(System.DefaultWorkingDirectory)/azurelinux + export BUILD_TYPE=${{ parameters.buildType }} + export BUILD_MACHINE=azurelinux + chmod +x $SOURCE_DIR/generate_package.sh + $SOURCE_DIR/generate_package.sh + displayName: "Run Package Script" + + # note: remove later. + - script: | + echo "=== Directory Structure ===" + find $(System.DefaultWorkingDirectory) -name "*.rpm" -type f + echo "=== STG_DIR Contents ===" + ls -la $(System.DefaultWorkingDirectory) + displayName: "Debug: List directory structure" + + - script: | + mkdir -p $(Build.ArtifactStagingDirectory)/artifacts/rpm + cp -f $(System.DefaultWorkingDirectory)/azurelinux/root/rpmbuild/RPMS/x86_64/aznfs-azurelinux-${{ parameters.versionName }}-1.x86_64.rpm $(Build.ArtifactStagingDirectory)/artifacts/rpm + echo "Listing Built Files..." + ls -R $(Build.ArtifactStagingDirectory) + displayName: "Copy RPM to Artifacts" + + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory) + artifactName: 'aznfs-temp' + - job: package_aznfs_arm64 displayName: Package and Release ${{ parameters.versionName }} for arm64 pool: