From 37adb69cc8060efd686121101488208de1a73643 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Mon, 29 Sep 2025 17:00:42 +0530 Subject: [PATCH 01/21] Initial changes for azurelinux pipeline setup devops --- .github/aznfs-build.yaml | 62 ++++++++ .github/workflows/prerelease.yml | 45 +++++- package.sh | 219 ++++++++++++++++----------- packaging/aznfs/RPM/aznfs.spec | 6 + turbonfs/CMakeLists.txt | 251 ++++++++++++++++++++----------- 5 files changed, 408 insertions(+), 175 deletions(-) diff --git a/.github/aznfs-build.yaml b/.github/aznfs-build.yaml index 5b4c92c1b..5a95c523e 100644 --- a/.github/aznfs-build.yaml +++ b/.github/aznfs-build.yaml @@ -55,6 +55,7 @@ stages: export STG_DIR=$(System.DefaultWorkingDirectory) export SOURCE_DIR=$(System.DefaultWorkingDirectory) export BUILD_TYPE=${{ parameters.buildType }} + export BUILD_MACHINE=ubuntu chmod +x $SOURCE_DIR/generate_package.sh $SOURCE_DIR/generate_package.sh displayName: "Run Package Script" @@ -79,6 +80,66 @@ stages: PathtoPublish: $(Build.ArtifactStagingDirectory) artifactName: 'aznfs-temp' + - job: package_aznfs_azurelinu_amd64 + displayName: Package and Release ${{ parameters.versionName }} for azurelinux + pool: + name: 'aznfsazurelinuxdemopool' # Use your self-hosted agent + + steps: + - checkout: self + path: azurelinux/AZNFS-mount + displayName: 'Checkout repository' + + - script: | + echo "Installing prerequisites on Azure Linux..." + sudo tdnf update -y + sudo tdnf install -y glibc-static rpm-build perl perl-IPC-Cmd \ + gcc gcc-c++ make git curl unzip zip tar cmake \ + zlib zlib-devel libuuid-devel gnutls gnutls-devel nfs-utils \ + libyaml-devel spdlog-devel nlohmann-json-devel cmake make \ + pkgconf pkgconf-m4 pkgconf-pkg-config ninja-build binutils \ + kernel-headers kernel-devel autoconf automake libtool jemalloc-devel + displayName: "Install prerequisites on Azure Linux" + + # note: remove later. + - script: | + echo "=== Validating Build Environment ===" + gcc --version + cmake --version + tdnf --version + echo "Architecture: $(uname -m)" + displayName: "Validate build environment" + + - script: | + export RELEASE_NUMBER=${{ parameters.versionName }} + export STG_DIR=$(System.DefaultWorkingDirectory) + export SOURCE_DIR=$(System.DefaultWorkingDirectory)/azurelinux + export BUILD_TYPE=${{ parameters.buildType }} + export BUILD_MACHINE=azurelinux + chmod +x $SOURCE_DIR/generate_package.sh + $SOURCE_DIR/generate_package.sh + displayName: "Run Package Script" + + # note: remove later. + - script: | + echo "=== Directory Structure ===" + find $(System.DefaultWorkingDirectory) -name "*.rpm" -type f + echo "=== STG_DIR Contents ===" + ls -la $(System.DefaultWorkingDirectory) + displayName: "Debug: List directory structure" + + - script: | + mkdir -p $(Build.ArtifactStagingDirectory)/artifacts/rpm + cp -f $(System.DefaultWorkingDirectory)/azurelinux/root/rpmbuild/RPMS/x86_64/aznfs-azurelinux-${{ parameters.versionName }}-1.x86_64.rpm $(Build.ArtifactStagingDirectory)/artifacts/rpm + echo "Listing Built Files..." + ls -R $(Build.ArtifactStagingDirectory) + displayName: "Copy RPM to Artifacts" + + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory) + artifactName: 'aznfs-temp' + - job: package_aznfs_arm64 displayName: Package and Release ${{ parameters.versionName }} for arm64 pool: @@ -103,6 +164,7 @@ stages: export STG_DIR=$(System.DefaultWorkingDirectory) export SOURCE_DIR=$(System.DefaultWorkingDirectory) export BUILD_TYPE=${{ parameters.buildType }} + export BUILD_MACHINE=ubuntu chmod +x $SOURCE_DIR/generate_package.sh $SOURCE_DIR/generate_package.sh displayName: "Run Package Script" diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index ba2da2b12..ce0508c9e 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -54,7 +54,8 @@ jobs: export RELEASE_NUMBER=${{ github.event.inputs.versionName }} export STG_DIR=$GITHUB_WORKSPACE/amd64 export SOURCE_DIR=$GITHUB_WORKSPACE/amd64 - export BUILD_TYPE=${{ github.events.inputs.buildType }} + export BUILD_TYPE=${{ github.event.inputs.buildType }} + export BUILD_MACHINE=ubuntu chmod +x $SOURCE_DIR/package.sh $SOURCE_DIR/package.sh - name: Create Test Release @@ -74,6 +75,46 @@ jobs: body: | New Test Release ${{ github.event.inputs.versionName }} + package_mount_helper_azurelinux: + name: Package and Release ${{ github.event.inputs.versionName }} for azurelinux + runs-on: self-hosted + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + path: azurelinux + + - name: Install prerequisites on Azure Linux + run: | + sudo tdnf update -y + sudo tdnf install -y glibc-static rpm-build perl perl-IPC-Cmd \ + gcc gcc-c++ make git curl unzip zip tar cmake \ + zlib zlib-devel libuuid-devel gnutls gnutls-devel nfs-utils \ + libyaml-devel spdlog-devel nlohmann-json-devel cmake make \ + pkgconf pkgconf-m4 pkgconf-pkg-config ninja-build binutils \ + kernel-headers kernel-devel autoconf automake libtool + + - name: Run Package.sh for azurelinux + run: | + export RELEASE_NUMBER=${{ github.event.inputs.versionName }} + export STG_DIR=$GITHUB_WORKSPACE + export SOURCE_DIR=$GITHUB_WORKSPACE/azurelinux + export BUILD_TYPE=${{ github.event.inputs.buildType }} + export BUILD_MACHINE=azurelinux + chmod +x $SOURCE_DIR/package.sh + $SOURCE_DIR/package.sh + + - name: Create Release (azurelinux) + uses: softprops/action-gh-release@v1 + with: + name: Release ${{ github.event.inputs.versionName }} + tag_name: ${{ github.event.inputs.versionName }} + target_commitish: ${{ github.sha }} + files: | + ${{ github.workspace }}/azurelinux/root/rpmbuild/RPMS/x86_64/aznfs-azurelinux-${{ github.event.inputs.versionName }}-1.x86_64.rpm + body: | + New Release ${{ github.event.inputs.versionName }} + package_mount_helper_arm64: name: Package and Release ${{ github.event.inputs.versionName }} for arm64 (Test Release) runs-on: ubuntu-22.04-arm @@ -101,7 +142,7 @@ jobs: export RELEASE_NUMBER=${{ github.event.inputs.versionName }} export STG_DIR=$GITHUB_WORKSPACE/arm64 export SOURCE_DIR=$GITHUB_WORKSPACE/arm64 - export BUILD_TYPE=${{ github.events.inputs.buildType }} + export BUILD_TYPE=${{ github.event.inputs.buildType }} chmod +x $SOURCE_DIR/package.sh $SOURCE_DIR/package.sh - name: Create Test Release diff --git a/package.sh b/package.sh index 55eeb2524..44cf82468 100755 --- a/package.sh +++ b/package.sh @@ -24,12 +24,19 @@ generate_rpm_package() { rpm_dir=$1 custom_stunnel_required=0 + azurelinux_build_required=0 # Overwrite rpm_pkg_dir in case of SUSE. if [ "$rpm_dir" == "suse" ]; then rpm_pkg_dir="${pkg_name}_sles-${RELEASE_NUMBER}-1.$arch" fi + # Overwrite rpm_pkg_dir in case of azurelinux. + if [ "$rpm_dir" == "azurelinux" ]; then + rpm_pkg_dir="${pkg_name}-azurelinux-${RELEASE_NUMBER}-1.$arch" + azurelinux_build_required=1 + fi + # Overwrite rpm_pkg_dir in case of Mariner, RedHat7, and Centos7. if [ "$rpm_dir" == "stunnel" ]; then rpm_pkg_dir="${pkg_name}_stunnel_custom-${RELEASE_NUMBER}-1.$arch" @@ -66,17 +73,33 @@ generate_rpm_package() # copy the aznfsclient config file. cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ - # copy the aznfsclient binary. - cp -avf ${aznfsclient} ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient + if [ "$rpm_dir" != "azurelinux" ]; then + # copy the aznfsclient binary. + cp -avf ${aznfsclient} ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient + else + # Define the final target location for aznfsclient + aznfsclient_target="${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient" - # - # Package aznfsclient dependencies in opt_dir/libs. - # libs_dir must already be populated with the required dependencies from - # the debian packaging step. Simply copy all those to rpm_libs_dir. - # - rpm_libs_dir=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/libs - mkdir -p ${rpm_libs_dir} - cp -avfH ${libs_dir}/* ${rpm_libs_dir} + # Copy the built aznfsclient binary to the target + cp -avf "${SOURCE_DIR}/turbonfs/build/aznfsclient" "${aznfsclient_target}" + + # Optional: fail early if copy fails + if [ ! -f "${aznfsclient_target}" ]; then + echo "Error: aznfsclient failed to copy to ${aznfsclient_target}" + exit 1 + fi + fi + + if [ "$rpm_dir" != "azurelinux" ]; then + # + # Package aznfsclient dependencies in opt_dir/libs. + # libs_dir must already be populated with the required dependencies from + # the debian packaging step. Simply copy all those to rpm_libs_dir. + # + rpm_libs_dir=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/libs + mkdir -p ${rpm_libs_dir} + cp -avfH ${libs_dir}/* ${rpm_libs_dir} + fi # Create the archive for the package. tar -cvzf ${STG_DIR}/${rpm_pkg_dir}.tar.gz -C ${STG_DIR}/${rpm_dir}/tmp root @@ -84,20 +107,22 @@ generate_rpm_package() # Copy the SPEC file to change the placeholders depending upon the RPM distro. cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/RPM/aznfs.spec ${STG_DIR}/${rpm_dir}/tmp/ - # - # Insert the contents of ${rpm_libs_dir}. - # This is variable due to the shared library versions. - # sed doesn't (easily) support replace target to be multi-line, so we use - # awk for this one. - # - opt_libs=$(for lib in ${rpm_libs_dir}/*; do echo ${opt_dir}/libs/$(basename $lib); done) - awk -i inplace -v r="$opt_libs" '{gsub(/OPT_LIBS/,r)}1' ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + if [ "$rpm_dir" != "azurelinux" ]; then + # + # Insert the contents of ${rpm_libs_dir}. + # This is variable due to the shared library versions. + # sed doesn't (easily) support replace target to be multi-line, so we use + # awk for this one. + # + opt_libs=$(for lib in ${rpm_libs_dir}/*; do echo ${opt_dir}/libs/$(basename $lib); done) + awk -i inplace -v r="$opt_libs" '{gsub(/OPT_LIBS/,r)}1' ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + fi # Insert current release number and RPM_DIR value. sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec sed -i -e "s/RPM_DIR/${rpm_dir}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec sed -i -e "s/BUILD_ARCH/${arch}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec - + # Replace the placeholders for various package names in aznfs.spec file. if [ "$rpm_dir" == "suse" ]; then sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}_sles/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec @@ -108,6 +133,8 @@ generate_rpm_package() else if [ "$rpm_dir" == "stunnel" ]; then sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}_stunnel_custom/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + elif [ "$rpm_dir" == "azurelinux" ]; then + sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}-azurelinux/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec else sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec fi @@ -121,7 +148,7 @@ generate_rpm_package() fi # Create the rpm package. - rpmbuild --define "custom_stunnel $custom_stunnel_required" --define "_topdir ${STG_DIR}/${rpm_dir}${rpmbuild_dir}" -v -bb ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + rpmbuild --define "custom_stunnel $custom_stunnel_required" --define "azurelinux_build $azurelinux_build_required" --define "_topdir ${STG_DIR}/${rpm_dir}${rpmbuild_dir}" -v -bb ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec } generate_tarball_package() @@ -203,29 +230,31 @@ rpm_buildroot_dir="${rpmbuild_dir}/BUILDROOT" # Insert release number to aznfs_install.sh sed -i -e "s/RELEASE_NUMBER=x.y.z/RELEASE_NUMBER=${RELEASE_NUMBER}/g" ${SOURCE_DIR}/scripts/aznfs_install.sh -######################### -# Generate .deb package # -######################### +if [ "$BUILD_MACHINE" != "azurelinux" ]; then + ######################### + # Generate .deb package # + ######################### -# Create the directory to hold the package control and data files for deb package. -mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN + # Create the directory to hold the package control and data files for deb package. + mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN -# Copy the debian control file(s) and maintainer scripts. -cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ -chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* + # Copy the debian control file(s) and maintainer scripts. + cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ + chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* -# Insert current release number. -sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control -sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control + # Insert current release number. + sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control + sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control -# Copy other static package file(s). -mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin -cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ -cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ + # Copy other static package file(s). + mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin + cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ + cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ -# Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. -mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin -gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs + # Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. + mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin + gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs +fi # # We build the turbonfs project here, note that we can set all cmake options in the @@ -246,6 +275,13 @@ else INSECURE_AUTH_FOR_DEVTEST=OFF fi +# Run azurelinux packaging only on azurelinux runner +if [ "$BUILD_MACHINE" == "azurelinux" ]; then + DYNAMIC_LINKS=ON +else + DYNAMIC_LINKS=OFF +fi + # vcpkg required env variable VCPKG_FORCE_SYSTEM_BINARIES to be set for arm64. if [ "$(uname -m)" == "aarch64" ]; then export VCPKG_FORCE_SYSTEM_BINARIES=1 @@ -254,72 +290,77 @@ fi cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DENABLE_PARANOID=${PARANOID} \ -DENABLE_INSECURE_AUTH_FOR_DEVTEST=${INSECURE_AUTH_FOR_DEVTEST} \ + -DENABLE_DYNAMIC_LINKS=${DYNAMIC_LINKS} \ -DPACKAGE_VERSION="${RELEASE_NUMBER}" \ -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake .. make popd -mkdir -p ${STG_DIR}/deb/${pkg_dir}${opt_dir} -cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/src/mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/src/nfsv3mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/src/nfsv4mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/scripts/aznfs_install.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/deb/${pkg_dir}/${opt_dir}/ +if [ "$BUILD_MACHINE" != "azurelinux" ]; then + mkdir -p ${STG_DIR}/deb/${pkg_dir}${opt_dir} + cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/nfsv3mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/nfsv4mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/scripts/aznfs_install.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/deb/${pkg_dir}/${opt_dir}/ -mkdir -p ${STG_DIR}/deb/${pkg_dir}${system_dir} -cp -avf ${SOURCE_DIR}/src/aznfswatchdog.service ${STG_DIR}/deb/${pkg_dir}${system_dir} -cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4.service ${STG_DIR}/deb/${pkg_dir}${system_dir} + mkdir -p ${STG_DIR}/deb/${pkg_dir}${system_dir} + cp -avf ${SOURCE_DIR}/src/aznfswatchdog.service ${STG_DIR}/deb/${pkg_dir}${system_dir} + cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4.service ${STG_DIR}/deb/${pkg_dir}${system_dir} -########################################### -# Bundle aznfsclient and its dependencies # -########################################### + ########################################### + # Bundle aznfsclient and its dependencies # + ########################################### -# aznfsclient in the final target dir. -aznfsclient=${STG_DIR}/deb/${pkg_dir}/sbin/aznfsclient -cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} + # aznfsclient in the final target dir. + aznfsclient=${STG_DIR}/deb/${pkg_dir}/sbin/aznfsclient + cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} -# Package aznfsclient dependencies in opt_dir. -libs_dir=${STG_DIR}/deb/${pkg_dir}${opt_dir}/libs -mkdir -p ${libs_dir} + # Package aznfsclient dependencies in opt_dir. + libs_dir=${STG_DIR}/deb/${pkg_dir}${opt_dir}/libs + mkdir -p ${libs_dir} -# Copy the dependencies. -cp -avfH $(ldd ${aznfsclient} | grep "=>" | awk '{print $3}') ${libs_dir} + # Copy the dependencies. + cp -avfH $(ldd ${aznfsclient} | grep "=>" | awk '{print $3}') ${libs_dir} -# -# Patch all the libs to reference shared libs from ${libs_dir}. -# This is our very simple containerization. -# -for lib in ${libs_dir}/*.so*; do - echo "Setting rpath to ${opt_dir}/libs for $lib" - patchelf --set-rpath ${opt_dir}/libs "$lib" -done + # + # Patch all the libs to reference shared libs from ${libs_dir}. + # This is our very simple containerization. + # + for lib in ${libs_dir}/*.so*; do + echo "Setting rpath to ${opt_dir}/libs for $lib" + patchelf --set-rpath ${opt_dir}/libs "$lib" + done -# -# Final containerization effort - bundle and use the same interpreter as the -# build machine. -# -ld_linux_path=$(ldd ${aznfsclient} | grep "ld-linux" | awk '{print $1}') -ld_linux_name=$(basename "$ld_linux_path") -ld_linux="${libs_dir}/${ld_linux_name}" -cp -avfH "${ld_linux_path}" "${ld_linux}" + # + # Final containerization effort - bundle and use the same interpreter as the + # build machine. + # + ld_linux_path=$(ldd ${aznfsclient} | grep "ld-linux" | awk '{print $1}') + ld_linux_name=$(basename "$ld_linux_path") + ld_linux="${libs_dir}/${ld_linux_name}" + cp -avfH "${ld_linux_path}" "${ld_linux}" -patchelf --set-interpreter ${opt_dir}/libs/${ld_linux_name} ${aznfsclient} + patchelf --set-interpreter ${opt_dir}/libs/${ld_linux_name} ${aznfsclient} -# Create the deb package. -dpkg-deb -Zgzip --root-owner-group --build $STG_DIR/deb/$pkg_dir + # Create the deb package. + dpkg-deb -Zgzip --root-owner-group --build $STG_DIR/deb/$pkg_dir -######################### -# Generate .rpm package # -######################### + ######################### + # Generate .rpm package # + ######################### -generate_rpm_package rpm -generate_rpm_package suse -# Generate rpm package with custom stunnel installation for Mariner, RedHat7, and Centos7. -generate_rpm_package stunnel + generate_rpm_package rpm + generate_rpm_package suse + # Generate rpm package with custom stunnel installation for Mariner, RedHat7, and Centos7. + generate_rpm_package stunnel -############################# -# Generating Tarball for AKS# -############################# + ############################# + # Generating Tarball for AKS# + ############################# -generate_tarball_package $arch + generate_tarball_package $arch +else + generate_rpm_package azurelinux +fi \ No newline at end of file diff --git a/packaging/aznfs/RPM/aznfs.spec b/packaging/aznfs/RPM/aznfs.spec index 8b4b381ec..11f086b3f 100644 --- a/packaging/aznfs/RPM/aznfs.spec +++ b/packaging/aznfs/RPM/aznfs.spec @@ -7,6 +7,10 @@ URL: https://github.com/Azure/AZNFS-mount/blob/main/README.md %if 0%{?custom_stunnel} Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, net-tools, binutils, kernel-headers, openssl, openssl-devel, gcc, make, wget Recommends: build-essential + +%elif 0%{?azurelinux_build} +Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, stunnel, net-tools, gnutls, jemalloc, libasan + %else Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, stunnel, net-tools %endif @@ -41,7 +45,9 @@ tar -xzvf ${STG_DIR}/AZNFS_PACKAGE_NAME-${RELEASE_NUMBER}-1.BUILD_ARCH.tar.gz -C /opt/microsoft/aznfs/aznfs_install.sh /lib/systemd/system/aznfswatchdog.service /lib/systemd/system/aznfswatchdogv4.service +%if !0%{?azurelinux_build} OPT_LIBS +%endif /opt/microsoft/aznfs/sample-turbo-config.yaml /sbin/aznfsclient diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index 6e8a554e8..ab94a6828 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -25,6 +25,9 @@ option(ENABLE_CHATTY "Enable super verbose logs" OFF) option(ENABLE_TCMALLOC "Use tcmalloc for malloc/free/new/delete" OFF) option(ENABLE_JEMALLOC "Use jemalloc for malloc/free/new/delete" ON) option(ENABLE_INSECURE_AUTH_FOR_DEVTEST "Enable AZAUTH for non-TLS connections" OFF) +option(ENABLE_DYNAMIC_LINKS "Use dynamic linking instead of bundling libs" OFF) + +message(STATUS "ENABLE_INSECURE_AUTH_FOR_DEVTEST = ${ENABLE_INSECURE_AUTH_FOR_DEVTEST}") # # Builds that make it to customers need to be extra careful about any unnecessary # logging. Some warning logs we have in our code are to attract developer @@ -144,16 +147,23 @@ if(NOT azure-storage-blobs-cpp_FOUND) message(FATAL_ERROR "azure-storage-blobs-cpp not found! Did you run cmake with -DCMAKE_TOOLCHAIN_FILE?") endif() +# Choose package manager based on ENABLE_DYNAMIC_LINKS +if(ENABLE_DYNAMIC_LINKS) + set(PKG_MGR tdnf) +else() + set(PKG_MGR apt) +endif() + # # Ensure pkg_search_module(), needed for configuring some packages. # find_package(PkgConfig QUIET) if(NOT PKG_CONFIG_FOUND) message(STATUS "pkg-config not found, trying to install pkg-config!") - execute_process(COMMAND sudo apt install -y pkg-config + execute_process(COMMAND sudo ${PKG_MGR} install -y pkg-config RESULT_VARIABLE PKGCONFIG_INSTALL_RESULT) if(NOT PKGCONFIG_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install pkg-config failed with ${PKGCONFIG_INSTALL_RESULT}, try installing pkg-config manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install pkg-config failed with ${PKGCONFIG_INSTALL_RESULT}, try installing pkg-config manually and then run cmake again") else() # Call once more to ensure above install completed fine and also # add the pkg_search_module() command. @@ -167,10 +177,10 @@ endif() find_program(MESON_EXECUTABLE meson QUIET) if(NOT MESON_FOUND) message(STATUS "meson not found, trying to install meson!") - execute_process(COMMAND sudo apt install -y meson + execute_process(COMMAND sudo ${PKG_MGR} install -y meson RESULT_VARIABLE MESON_INSTALL_RESULT) if(NOT MESON_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install meson failed with ${MESON_INSTALL_RESULT}, try installing meson manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install meson failed with ${MESON_INSTALL_RESULT}, try installing meson manually and then run cmake again") else() find_program(MESON_EXECUTABLE meson REQUIRED) endif() @@ -182,19 +192,31 @@ endif() find_program(NINJA_EXECUTABLE ninja QUIET) if(NOT NINJA_FOUND) message(STATUS "ninja not found, trying to install ninja-build!") - execute_process(COMMAND sudo apt install -y ninja-build + execute_process(COMMAND sudo ${PKG_MGR} install -y ninja-build RESULT_VARIABLE NINJA_INSTALL_RESULT) if(NOT NINJA_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install ninja-build failed with ${NINJA_INSTALL_RESULT}, try installing ninja-build manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ninja-build failed with ${NINJA_INSTALL_RESULT}, try installing ninja-build manually and then run cmake again") else() find_program(NINJA_EXECUTABLE ninja REQUIRED) endif() endif() -# Meson will install the fuse library and header files in the following paths. -set(fuse3_LIBRARY "/usr/local/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/libfuse3.a") +# Decide fuse3 paths based on ENABLE_DYNAMIC_LINKS. +if(ENABLE_DYNAMIC_LINKS) + # Dynamic: Meson installs directly to /usr/local/lib. + set(fuse3_LIBRARY "/usr/local/lib/libfuse3.a") +else() + # Static: Use multiarch layout. + set(fuse3_LIBRARY "/usr/local/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/libfuse3.a") +endif() + +# Include path remains the same for both cases. set(fuse3_INCLUDE_DIR "/usr/local/include") +# Meson will install the fuse library and header files in the following paths. +# set(fuse3_LIBRARY "/usr/local/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/libfuse3.a") +# set(fuse3_INCLUDE_DIR "/usr/local/include") + add_custom_command( OUTPUT ${fuse3_LIBRARY} COMMAND ${MESON_EXECUTABLE} setup ${LIBFUSE_BUILD_DIR} ${LIBFUSE_SOURCE_DIR} --default-library=static --buildtype=${MESON_BUILD_TYPE} @@ -210,10 +232,15 @@ add_custom_target(libfuse ALL DEPENDS ${fuse3_LIBRARY}) find_package(GnuTLS "3.4.6" QUIET) if(NOT GNUTLS_FOUND) message(STATUS "GnuTLS not found, trying to install gnutls-dev!") - execute_process(COMMAND sudo apt install -y gnutls-dev + if(PKG_MGR STREQUAL "apt") + set(GNUTLS_PKG "gnutls-dev") + else() + set(GNUTLS_PKG "gnutls-devel") + endif() + execute_process(COMMAND sudo ${PKG_MGR} install -y ${GNUTLS_PKG} RESULT_VARIABLE GNUTLS_INSTALL_RESULT) if(NOT GNUTLS_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install gnutls-dev failed with ${GNUTLS_INSTALL_RESULT}, try installing gnutls-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${GNUTLS_PKG} failed with ${GNUTLS_INSTALL_RESULT}, try installing ${GNUTLS_PKG} manually and then run cmake again") else() find_package(GnuTLS "3.4.6" REQUIRED) endif() @@ -229,11 +256,16 @@ endif() find_package(tcmalloc QUIET) if(NOT tcmalloc_FOUND) - message(STATUS "tcmalloc not found, trying to install libgoogle-perftools-dev!") - execute_process(COMMAND sudo apt install -y libgoogle-perftools-dev + if(PKG_MGR STREQUAL "apt") + set(TCMALLOC_PKG "libgoogle-perftools-dev") + else() + set(TCMALLOC_PKG "gperftools-devel") + endif() + message(STATUS "tcmalloc not found, trying to install ${TCMALLOC_PKG}!") + execute_process(COMMAND sudo ${PKG_MGR} install -y ${TCMALLOC_PKG} RESULT_VARIABLE tcmalloc_INSTALL_RESULT) if(NOT tcmalloc_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install libgoogle-perftools-dev failed with ${tcmalloc_INSTALL_RESULT}, try installing libgoogle-perftools-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${TCMALLOC_PKG} failed with ${tcmalloc_INSTALL_RESULT}, try installing ${TCMALLOC_PKG} manually and then run cmake again") else() # Call once more to ensure above install completed fine and also # will set tcmalloc_INCLUDE_DIR and tcmalloc_LIBRARY variables. @@ -256,20 +288,29 @@ endif() pkg_search_module(JEMALLOC QUIET jemalloc) if(NOT JEMALLOC_FOUND) - message(STATUS "jemalloc not found, trying to install libjemalloc-dev!") - execute_process(COMMAND sudo apt install -y libjemalloc-dev + if(${PKG_MGR} STREQUAL "tdnf") + set(JEMALLOC_PKG jemalloc-devel) + else() + set(JEMALLOC_PKG libjemalloc-dev) + endif() + message(STATUS "jemalloc not found, trying to install ${JEMALLOC_PKG}!") + execute_process(COMMAND sudo ${PKG_MGR} install -y ${JEMALLOC_PKG} RESULT_VARIABLE JEMALLOC_INSTALL_RESULT) if(NOT JEMALLOC_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install libjemalloc-dev failed with ${JEMALLOC_INSTALL_RESULT}, try installing libjemalloc-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${JEMALLOC_PKG} failed with ${JEMALLOC_INSTALL_RESULT}, try installing ${JEMALLOC_PKG} manually and then run cmake again") else() # Call once more to ensure above install completed fine and also - # will set JEMALLOC_LIBRARY_DIRS variable. + # will set JEMALLOC_LIBRARIES variable. pkg_search_module(JEMALLOC REQUIRED jemalloc) endif() endif() -# We want to link against the static jemalloc lib. -message(STATUS "Using jemalloc static lib ${JEMALLOC_LIBRARY_DIRS}/libjemalloc.a") +if(ENABLE_DYNAMIC_LINKS) + message(STATUS "Using jemalloc lib ${JEMALLOC_LIBRARIES}") +else() + # We want to link against the static jemalloc lib. + message(STATUS "Using jemalloc static lib ${JEMALLOC_LIBRARY_DIRS}") +endif() endif() # @@ -277,11 +318,16 @@ endif() # pkg_search_module(UUID QUIET uuid) if(NOT UUID_FOUND) - message(STATUS "uuid not found, trying to install uuid-dev!") - execute_process(COMMAND sudo apt install -y uuid-dev + if(PKG_MGR STREQUAL "apt") + set(UUID_PKG "uuid-dev") + else() + set(UUID_PKG "uuid-devel") + endif() + message(STATUS "uuid not found, trying to install ${UUID_PKG}!") + execute_process(COMMAND sudo ${PKG_MGR} install -y ${UUID_PKG} RESULT_VARIABLE UUID_INSTALL_RESULT) if(NOT UUID_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install uuid-dev failed with ${UUID_INSTALL_RESULT}, try installing uuid-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${UUID_PKG} failed with ${UUID_INSTALL_RESULT}, try installing ${UUID_PKG} manually and then run cmake again") else() pkg_search_module(UUID REQUIRED uuid) endif() @@ -295,11 +341,16 @@ message(STATUS "Using uuid include dir ${UUID_INCLUDE_DIRS}") # find_package(ZLIB QUIET) if(NOT ZLIB_FOUND) - message(STATUS "zlib not found, trying to install zlib1g-dev!") - execute_process(COMMAND sudo apt install -y zlib1g-dev + if(PKG_MGR STREQUAL "apt") + set(ZLIB_PKG "zlib1g-dev") + else() + set(ZLIB_PKG "zlib-devel") + endif() + message(STATUS "zlib not found, trying to install ${ZLIB_PKG}!") + execute_process(COMMAND sudo ${PKG_MGR} install -y ${ZLIB_PKG} RESULT_VARIABLE ZLIB_INSTALL_RESULT) if(NOT ZLIB_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install zlib1g-dev failed with ${ZLIB_INSTALL_RESULT}, try installing zlib1g-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${ZLIB_PKG} failed with ${ZLIB_INSTALL_RESULT}, try installing ${ZLIB_PKG} manually and then run cmake again") else() # Call once more to ensure above install completed fine and also # will set ZLIB_INCLUDE_DIR and ZLIB_LIBRARY variables. @@ -377,66 +428,85 @@ target_compile_options(${CMAKE_PROJECT_NAME} PRIVATE -Werror ) -# -# Libraries for statically linking gnutls. -# libp11-kit and libunistring do not have a static version available, so we are -# forced to use the shared version and bundle it. -# -set(CMAKE_FIND_LIBRARY_SUFFIXES .a) +# Only use static gnutls linking when ENABLE_DYNAMIC_LINKS is NOT set. +if(NOT ENABLE_DYNAMIC_LINKS) + # + # Libraries for statically linking gnutls. + # libp11-kit and libunistring do not have a static version available, so we are + # forced to use the shared version and bundle it. + # + set(CMAKE_FIND_LIBRARY_SUFFIXES .a) + + find_library(GNUTLS_STATIC_LIB libgnutls.a REQUIRED) + # + # arm64 libgnutls.a has multiple definitions for some symbols exported by + # libcrypto.a. The libcrypto.a object has some more symbols which are needed, + # so we delete the offending object from libgnutls.a before linking. + # + if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") + execute_process( + COMMAND bash -c "cp -vf ${GNUTLS_STATIC_LIB} /tmp/gnutls.a; ar dv /tmp/gnutls.a aes-aarch64.o" + COMMAND_ERROR_IS_FATAL ANY) + set(GNUTLS_STATIC_LIB /tmp/gnutls.a) + endif() -find_library(GNUTLS_STATIC_LIB libgnutls.a REQUIRED) -# -# arm64 libgnutls.a has multiple definitions for some symbols exported by -# libcrypto.a. The libcrypto.a object has some more symbols which are needed, -# so we delete the offending object from libgnutls.a before linking. -# -if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") -execute_process( - COMMAND bash -c "cp -vf ${GNUTLS_STATIC_LIB} /tmp/gnutls.a; ar dv /tmp/gnutls.a aes-aarch64.o" - COMMAND_ERROR_IS_FATAL ANY) -set(GNUTLS_STATIC_LIB /tmp/gnutls.a) -endif() - -find_library(HOGWEED_STATIC_LIB libhogweed.a REQUIRED) -find_library(NETTLE_STATIC_LIB libnettle.a REQUIRED) -find_library(TASN1_STATIC_LIB libtasn1.a REQUIRED) -find_library(IDN2_STATIC_LIB libidn2.a REQUIRED) -find_library(GMP_STATIC_LIB libgmp.a REQUIRED) -set(CMAKE_FIND_LIBRARY_SUFFIXES .so) -find_library(P11_KIT_SHARED_LIB libp11-kit.so REQUIRED) -find_library(UNISTRING_SHARED_LIB NAMES - libunistring.so - libunistring.so.5 - libunistring.so.2 - REQUIRED) - -set(GNUTLS_ALL_LIBRARIES - ${GNUTLS_STATIC_LIB} - ${HOGWEED_STATIC_LIB} - ${NETTLE_STATIC_LIB} - ${TASN1_STATIC_LIB} - ${IDN2_STATIC_LIB} - ${GMP_STATIC_LIB} - ${P11_KIT_SHARED_LIB} - ${UNISTRING_SHARED_LIB}) -message(STATUS "Using gnutls libraries: ${GNUTLS_ALL_LIBRARIES}") + find_library(HOGWEED_STATIC_LIB libhogweed.a REQUIRED) + find_library(NETTLE_STATIC_LIB libnettle.a REQUIRED) + find_library(TASN1_STATIC_LIB libtasn1.a REQUIRED) + find_library(IDN2_STATIC_LIB libidn2.a REQUIRED) + find_library(GMP_STATIC_LIB libgmp.a REQUIRED) + set(CMAKE_FIND_LIBRARY_SUFFIXES .so) + find_library(P11_KIT_SHARED_LIB libp11-kit.so REQUIRED) + find_library(UNISTRING_SHARED_LIB NAMES + libunistring.so + libunistring.so.5 + libunistring.so.2 + REQUIRED) + + set(GNUTLS_ALL_LIBRARIES + ${GNUTLS_STATIC_LIB} + ${HOGWEED_STATIC_LIB} + ${NETTLE_STATIC_LIB} + ${TASN1_STATIC_LIB} + ${IDN2_STATIC_LIB} + ${GMP_STATIC_LIB} + ${P11_KIT_SHARED_LIB} + ${UNISTRING_SHARED_LIB}) + message(STATUS "Using gnutls libraries: ${GNUTLS_ALL_LIBRARIES}") +endif() # All libraries. -target_link_libraries(${CMAKE_PROJECT_NAME} - -static-libgcc -static-libstdc++ - -Wl,-rpath,/opt/microsoft/aznfs/libs - ${ZLIB_LIBRARIES} - dl - pthread - nfs - # GNUTLS libraries are needed by static libnfs. - ${GNUTLS_ALL_LIBRARIES} - yaml-cpp - spdlog - Azure::azure-identity - Azure::azure-storage-blobs - nlohmann_json::nlohmann_json - uuid) +if(ENABLE_DYNAMIC_LINKS) + # Dynamic linking: No static flags, use GnuTLS::GnuTLS target + target_link_libraries(${CMAKE_PROJECT_NAME} + ${ZLIB_LIBRARIES} + dl + pthread + nfs + yaml-cpp + spdlog + Azure::azure-identity + Azure::azure-storage-blobs + nlohmann_json::nlohmann_json + uuid + GnuTLS::GnuTLS) +else() + # Static linking: Use static-libgcc/libstdc++ and explicit gnutls libraries + target_link_libraries(${CMAKE_PROJECT_NAME} + -static-libgcc -static-libstdc++ + -Wl,-rpath,/opt/microsoft/aznfs/libs + ${ZLIB_LIBRARIES} + dl + pthread + nfs + ${GNUTLS_ALL_LIBRARIES} + yaml-cpp + spdlog + Azure::azure-identity + Azure::azure-storage-blobs + nlohmann_json::nlohmann_json + uuid) +endif() if(ENABLE_TCMALLOC) target_link_libraries(${CMAKE_PROJECT_NAME} @@ -444,8 +514,21 @@ target_link_libraries(${CMAKE_PROJECT_NAME} endif() if(ENABLE_JEMALLOC) -target_link_libraries(${CMAKE_PROJECT_NAME} - ${JEMALLOC_LIBRARY_DIRS}/libjemalloc.a) + if(ENABLE_DYNAMIC_LINKS) + target_link_libraries(${CMAKE_PROJECT_NAME} + ${JEMALLOC_LIBRARIES}) + else() + target_link_libraries(${CMAKE_PROJECT_NAME} + ${JEMALLOC_LIBRARY_DIRS}/libjemalloc.a) + endif() endif() +# --- Debug: Print all static libraries linked --- +get_target_property(LINK_LIBS ${CMAKE_PROJECT_NAME} LINK_LIBRARIES) +message(STATUS "==== Libraries linked to ${CMAKE_PROJECT_NAME} ====") +foreach(lib IN LISTS LINK_LIBS) + message(STATUS " ${lib}") +endforeach() +message(STATUS "===============================================") + install(TARGETS ${CMAKE_PROJECT_NAME}) From 04d674699b6da856a35ec888b5c4b295cc081341 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Mon, 29 Sep 2025 17:45:36 +0530 Subject: [PATCH 02/21] Initial changes for azurelinux pipeline setup devops --- .github/aznfs-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/aznfs-build.yaml b/.github/aznfs-build.yaml index 5a95c523e..bfe15cd9d 100644 --- a/.github/aznfs-build.yaml +++ b/.github/aznfs-build.yaml @@ -113,7 +113,7 @@ stages: - script: | export RELEASE_NUMBER=${{ parameters.versionName }} export STG_DIR=$(System.DefaultWorkingDirectory) - export SOURCE_DIR=$(System.DefaultWorkingDirectory)/azurelinux + export SOURCE_DIR=$(System.DefaultWorkingDirectory) export BUILD_TYPE=${{ parameters.buildType }} export BUILD_MACHINE=azurelinux chmod +x $SOURCE_DIR/generate_package.sh From a34d5ed521e73a9038eefa0f12f56f1b5e57b74c Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Mon, 29 Sep 2025 18:32:22 +0530 Subject: [PATCH 03/21] Initial changes for azurelinux pipeline setup devops --- generate_package.sh | 213 +++++++++++++++++++++++++++----------------- 1 file changed, 129 insertions(+), 84 deletions(-) diff --git a/generate_package.sh b/generate_package.sh index 54d02649f..c545ba23f 100644 --- a/generate_package.sh +++ b/generate_package.sh @@ -24,12 +24,19 @@ generate_rpm_package() { rpm_dir=$1 custom_stunnel_required=0 + azurelinux_build_required=0 # Overwrite rpm_pkg_dir in case of RedHat7 and Centos7. if [ "$rpm_dir" == "stunnel" ]; then custom_stunnel_required=1 fi + # Overwrite rpm_pkg_dir in case of azurelinux. + if [ "$rpm_dir" == "azurelinux" ]; then + rpm_pkg_dir="${pkg_name}-azurelinux-${RELEASE_NUMBER}-1.$arch" + azurelinux_build_required=1 + fi + # Create the directory to hold the package spec and data files for RPM package. mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir} @@ -60,17 +67,33 @@ generate_rpm_package() # copy the aznfsclient config file. cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ - # copy the aznfsclient binary. - cp -avf ${aznfsclient} ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient + if [ "$rpm_dir" != "azurelinux" ]; then + # copy the aznfsclient binary. + cp -avf ${aznfsclient} ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient + else + # Define the final target location for aznfsclient + aznfsclient_target="${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient" - # - # Package aznfsclient dependencies in opt_dir/libs. - # libs_dir must already be populated with the required dependencies from - # the debian packaging step. Simply copy all those to rpm_libs_dir. - # - rpm_libs_dir=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/libs - mkdir -p ${rpm_libs_dir} - cp -avfH ${libs_dir}/* ${rpm_libs_dir} + # Copy the built aznfsclient binary to the target + cp -avf "${SOURCE_DIR}/turbonfs/build/aznfsclient" "${aznfsclient_target}" + + # Optional: fail early if copy fails + if [ ! -f "${aznfsclient_target}" ]; then + echo "Error: aznfsclient failed to copy to ${aznfsclient_target}" + exit 1 + fi + fi + + if [ "$rpm_dir" != "azurelinux" ]; then + # + # Package aznfsclient dependencies in opt_dir/libs. + # libs_dir must already be populated with the required dependencies from + # the debian packaging step. Simply copy all those to rpm_libs_dir. + # + rpm_libs_dir=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/libs + mkdir -p ${rpm_libs_dir} + cp -avfH ${libs_dir}/* ${rpm_libs_dir} + fi # Create the archive for the package. tar -cvzf ${STG_DIR}/${rpm_pkg_dir}.tar.gz -C ${STG_DIR}/${rpm_dir}/tmp root @@ -78,14 +101,16 @@ generate_rpm_package() # Copy the SPEC file to change the placeholders depending upon the RPM distro. cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/RPM/aznfs.spec ${STG_DIR}/${rpm_dir}/tmp/ - # - # Insert the contents of ${rpm_libs_dir}. - # This is variable due to the shared library versions. - # sed doesn't (easily) support replace target to be multi-line, so we use - # awk for this one. - # - opt_libs=$(for lib in ${rpm_libs_dir}/*; do echo ${opt_dir}/libs/$(basename $lib); done) - awk -i inplace -v r="$opt_libs" '{gsub(/OPT_LIBS/,r)}1' ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + if [ "$rpm_dir" != "azurelinux" ]; then + # + # Insert the contents of ${rpm_libs_dir}. + # This is variable due to the shared library versions. + # sed doesn't (easily) support replace target to be multi-line, so we use + # awk for this one. + # + opt_libs=$(for lib in ${rpm_libs_dir}/*; do echo ${opt_dir}/libs/$(basename $lib); done) + awk -i inplace -v r="$opt_libs" '{gsub(/OPT_LIBS/,r)}1' ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + fi # Insert current release number and RPM_DIR value. sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec @@ -100,6 +125,10 @@ generate_rpm_package() sed -i -e "s/PROCPS_PACKAGE_NAME/sysvinit-tools/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec sed -i -e "s/DISTRO/suse/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec else + if [ "$rpm_dir" == "azurelinux" ]; then + sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}-azurelinux/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + fi + sed -i -e "s/NETCAT_PACKAGE_NAME/nmap-ncat/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec # In new versions of Centos/RedHat/Rocky, procps-ng provides pidof. For older versions, it is provided by sysvinit-tools but since it is not # present in new versions, only install procps-ng which exists in all versions. @@ -109,7 +138,7 @@ generate_rpm_package() fi # Create the rpm package. - rpmbuild --define "custom_stunnel $custom_stunnel_required" --define "_topdir ${STG_DIR}/${rpm_dir}${rpmbuild_dir}" -v -bb ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + rpmbuild --define "custom_stunnel $custom_stunnel_required" --define "azurelinux_build $azurelinux_build_required" --define "_topdir ${STG_DIR}/${rpm_dir}${rpmbuild_dir}" -v -bb ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec # Remove the temporary files. rm ${STG_DIR}/${rpm_pkg_dir}.tar.gz @@ -128,29 +157,33 @@ rpm_buildroot_dir="${rpmbuild_dir}/BUILDROOT" # Insert release number to aznfs_install.sh sed -i -e "s/RELEASE_NUMBER=x.y.z/RELEASE_NUMBER=${RELEASE_NUMBER}/g" ${SOURCE_DIR}/scripts/aznfs_install.sh -######################### -# Generate .deb package # -######################### -# Create the directory to hold the package control and data files for deb package. -mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN +if [ "$BUILD_MACHINE" != "azurelinux" ]; then + ######################### + # Generate .deb package # + ######################### + + # Create the directory to hold the package control and data files for deb package. + mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN -# Copy the debian control file(s) and maintainer scripts. -cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ -chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* + # Copy the debian control file(s) and maintainer scripts. + cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ + chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* -# Insert current release number. -sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control -sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control + # Insert current release number. + sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control + sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control -# Copy other static package file(s). -mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin -cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ -cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ + # Copy other static package file(s). + mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin + cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ + cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ + + # Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. + mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin + gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs +fi -# Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. -mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin -gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs # # We build the turbonfs project here, note that we can set all cmake options in the @@ -171,6 +204,13 @@ else INSECURE_AUTH_FOR_DEVTEST=OFF fi +# Run azurelinux packaging only on azurelinux runner +if [ "$BUILD_MACHINE" == "azurelinux" ]; then + DYNAMIC_LINKS=ON +else + DYNAMIC_LINKS=OFF +fi + # vcpkg required env variable VCPKG_FORCE_SYSTEM_BINARIES to be set for arm64. if [ "$(uname -m)" == "aarch64" ]; then export VCPKG_FORCE_SYSTEM_BINARIES=1 @@ -179,68 +219,73 @@ fi cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DENABLE_PARANOID=${PARANOID} \ -DENABLE_INSECURE_AUTH_FOR_DEVTEST=${INSECURE_AUTH_FOR_DEVTEST} \ + -DENABLE_DYNAMIC_LINKS=${DYNAMIC_LINKS} \ -DPACKAGE_VERSION="${RELEASE_NUMBER}" \ -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake .. make popd -mkdir -p ${STG_DIR}/deb/${pkg_dir}${opt_dir} -cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/src/mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/src/nfsv3mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/src/nfsv4mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/scripts/aznfs_install.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/deb/${pkg_dir}/${opt_dir}/ +if [ "$BUILD_MACHINE" != "azurelinux" ]; then + mkdir -p ${STG_DIR}/deb/${pkg_dir}${opt_dir} + cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/nfsv3mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/nfsv4mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/scripts/aznfs_install.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/deb/${pkg_dir}/${opt_dir}/ -mkdir -p ${STG_DIR}/deb/${pkg_dir}${system_dir} -cp -avf ${SOURCE_DIR}/src/aznfswatchdog.service ${STG_DIR}/deb/${pkg_dir}${system_dir} -cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4.service ${STG_DIR}/deb/${pkg_dir}${system_dir} + mkdir -p ${STG_DIR}/deb/${pkg_dir}${system_dir} + cp -avf ${SOURCE_DIR}/src/aznfswatchdog.service ${STG_DIR}/deb/${pkg_dir}${system_dir} + cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4.service ${STG_DIR}/deb/${pkg_dir}${system_dir} -########################################### -# Bundle aznfsclient and its dependencies # -########################################### + ########################################### + # Bundle aznfsclient and its dependencies # + ########################################### -# aznfsclient in the final target dir. -aznfsclient=${STG_DIR}/deb/${pkg_dir}/sbin/aznfsclient -cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} + # aznfsclient in the final target dir. + aznfsclient=${STG_DIR}/deb/${pkg_dir}/sbin/aznfsclient + cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} -# Package aznfsclient dependencies in opt_dir. -libs_dir=${STG_DIR}/deb/${pkg_dir}${opt_dir}/libs -mkdir -p ${libs_dir} + # Package aznfsclient dependencies in opt_dir. + libs_dir=${STG_DIR}/deb/${pkg_dir}${opt_dir}/libs + mkdir -p ${libs_dir} -# Copy the dependencies. -cp -avfH $(ldd ${aznfsclient} | grep "=>" | awk '{print $3}') ${libs_dir} + # Copy the dependencies. + cp -avfH $(ldd ${aznfsclient} | grep "=>" | awk '{print $3}') ${libs_dir} -# -# Patch all the libs to reference shared libs from ${libs_dir}. -# This is our very simple containerization. -# -for lib in ${libs_dir}/*.so*; do - echo "Setting rpath to ${opt_dir}/libs for $lib" - patchelf --set-rpath ${opt_dir}/libs "$lib" -done + # + # Patch all the libs to reference shared libs from ${libs_dir}. + # This is our very simple containerization. + # + for lib in ${libs_dir}/*.so*; do + echo "Setting rpath to ${opt_dir}/libs for $lib" + patchelf --set-rpath ${opt_dir}/libs "$lib" + done -# -# Final containerization effort - bundle and use the same interpreter as the -# build machine. -# -ld_linux_path=$(ldd ${aznfsclient} | grep "ld-linux" | awk '{print $1}') -ld_linux_name=$(basename "$ld_linux_path") -ld_linux="${libs_dir}/${ld_linux_name}" -cp -avfH "${ld_linux_path}" "${ld_linux}" + # + # Final containerization effort - bundle and use the same interpreter as the + # build machine. + # + ld_linux_path=$(ldd ${aznfsclient} | grep "ld-linux" | awk '{print $1}') + ld_linux_name=$(basename "$ld_linux_path") + ld_linux="${libs_dir}/${ld_linux_name}" + cp -avfH "${ld_linux_path}" "${ld_linux}" -patchelf --set-interpreter ${opt_dir}/libs/${ld_linux_name} ${aznfsclient} + patchelf --set-interpreter ${opt_dir}/libs/${ld_linux_name} ${aznfsclient} -# Create the deb package. -dpkg-deb -Zgzip --root-owner-group --build $STG_DIR/deb/$pkg_dir + # Create the deb package. + dpkg-deb -Zgzip --root-owner-group --build $STG_DIR/deb/$pkg_dir -######################### -# Generate .rpm package # -######################### + ######################### + # Generate .rpm package # + ######################### -generate_rpm_package rpm -generate_rpm_package suse -# Generate rpm package with custom stunnel installation for RedHat7 and Centos7. -generate_rpm_package stunnel + generate_rpm_package rpm + generate_rpm_package suse + # Generate rpm package with custom stunnel installation for RedHat7 and Centos7. + generate_rpm_package stunnel +else + generate_rpm_package azurelinux +fi From a5248ac03a9ba74820b82325982c8d1434c95fbd Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Mon, 29 Sep 2025 20:12:33 +0530 Subject: [PATCH 04/21] Initial changes for azurelinux pipeline setup devops --- generate_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate_package.sh b/generate_package.sh index c545ba23f..a8bbc7c6f 100644 --- a/generate_package.sh +++ b/generate_package.sh @@ -33,7 +33,7 @@ generate_rpm_package() # Overwrite rpm_pkg_dir in case of azurelinux. if [ "$rpm_dir" == "azurelinux" ]; then - rpm_pkg_dir="${pkg_name}-azurelinux-${RELEASE_NUMBER}-1.$arch" + # rpm_pkg_dir="${pkg_name}-azurelinux-${RELEASE_NUMBER}-1.$arch" azurelinux_build_required=1 fi From 9c9aec7ff21ac54c3acbe7433f32ef54d3f29817 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Tue, 30 Sep 2025 13:44:59 +0530 Subject: [PATCH 05/21] Initial changes for azurelinux pipeline setup devops2 --- .github/aznfs-build.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/aznfs-build.yaml b/.github/aznfs-build.yaml index bfe15cd9d..053f548c7 100644 --- a/.github/aznfs-build.yaml +++ b/.github/aznfs-build.yaml @@ -116,7 +116,7 @@ stages: export SOURCE_DIR=$(System.DefaultWorkingDirectory) export BUILD_TYPE=${{ parameters.buildType }} export BUILD_MACHINE=azurelinux - chmod +x $SOURCE_DIR/generate_package.sh + chmod +x $SOURCE_DIR/generate_package.sh $SOURCE_DIR/generate_package.sh displayName: "Run Package Script" @@ -129,8 +129,8 @@ stages: displayName: "Debug: List directory structure" - script: | - mkdir -p $(Build.ArtifactStagingDirectory)/artifacts/rpm - cp -f $(System.DefaultWorkingDirectory)/azurelinux/root/rpmbuild/RPMS/x86_64/aznfs-azurelinux-${{ parameters.versionName }}-1.x86_64.rpm $(Build.ArtifactStagingDirectory)/artifacts/rpm + mkdir -p $(Build.ArtifactStagingDirectory)/artifacts/rpmazurelinux + cp -f $(System.DefaultWorkingDirectory)/azurelinux/root/rpmbuild/RPMS/x86_64/aznfs-${{ parameters.versionName }}-1.x86_64.rpm $(Build.ArtifactStagingDirectory)/artifacts/rpmazurelinux echo "Listing Built Files..." ls -R $(Build.ArtifactStagingDirectory) displayName: "Copy RPM to Artifacts" From 8c17c68a5977dcae34fd2dd276d28a5e15517e85 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Wed, 1 Oct 2025 04:30:52 +0530 Subject: [PATCH 06/21] Initial changes for azurelinux pipeline setup devops2 --- .github/aznfs-build.yaml | 1 + .github/workflows/prerelease.yml | 74 ++++++++++++++++---------------- 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/.github/aznfs-build.yaml b/.github/aznfs-build.yaml index 053f548c7..ae0eb3e17 100644 --- a/.github/aznfs-build.yaml +++ b/.github/aznfs-build.yaml @@ -188,6 +188,7 @@ stages: PathtoPublish: $(Build.ArtifactStagingDirectory) artifactName: 'aznfs-temp' +# TODO: add azurelinux later to release and publish stages post approvals - stage: ReleaseArtifacts displayName: "Sign and Release Artifacts" jobs: diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index ce0508c9e..5d88b0f91 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -75,45 +75,47 @@ jobs: body: | New Test Release ${{ github.event.inputs.versionName }} - package_mount_helper_azurelinux: - name: Package and Release ${{ github.event.inputs.versionName }} for azurelinux - runs-on: self-hosted - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - path: azurelinux + # NOTE: UNCOMMENT THIS JOB ONCE AZURE LINUX RUNNER IS CONNECTED + # + # package_mount_helper_azurelinux: + # name: Package and Release ${{ github.event.inputs.versionName }} for azurelinux + # runs-on: self-hosted + # steps: + # - name: Checkout repository + # uses: actions/checkout@v3 + # with: + # path: azurelinux - - name: Install prerequisites on Azure Linux - run: | - sudo tdnf update -y - sudo tdnf install -y glibc-static rpm-build perl perl-IPC-Cmd \ - gcc gcc-c++ make git curl unzip zip tar cmake \ - zlib zlib-devel libuuid-devel gnutls gnutls-devel nfs-utils \ - libyaml-devel spdlog-devel nlohmann-json-devel cmake make \ - pkgconf pkgconf-m4 pkgconf-pkg-config ninja-build binutils \ - kernel-headers kernel-devel autoconf automake libtool + # - name: Install prerequisites on Azure Linux + # run: | + # sudo tdnf update -y + # sudo tdnf install -y glibc-static rpm-build perl perl-IPC-Cmd \ + # gcc gcc-c++ make git curl unzip zip tar cmake \ + # zlib zlib-devel libuuid-devel gnutls gnutls-devel nfs-utils \ + # libyaml-devel spdlog-devel nlohmann-json-devel cmake make \ + # pkgconf pkgconf-m4 pkgconf-pkg-config ninja-build binutils \ + # kernel-headers kernel-devel autoconf automake libtool - - name: Run Package.sh for azurelinux - run: | - export RELEASE_NUMBER=${{ github.event.inputs.versionName }} - export STG_DIR=$GITHUB_WORKSPACE - export SOURCE_DIR=$GITHUB_WORKSPACE/azurelinux - export BUILD_TYPE=${{ github.event.inputs.buildType }} - export BUILD_MACHINE=azurelinux - chmod +x $SOURCE_DIR/package.sh - $SOURCE_DIR/package.sh + # - name: Run Package.sh for azurelinux + # run: | + # export RELEASE_NUMBER=${{ github.event.inputs.versionName }} + # export STG_DIR=$GITHUB_WORKSPACE + # export SOURCE_DIR=$GITHUB_WORKSPACE/azurelinux + # export BUILD_TYPE=${{ github.event.inputs.buildType }} + # export BUILD_MACHINE=azurelinux + # chmod +x $SOURCE_DIR/package.sh + # $SOURCE_DIR/package.sh - - name: Create Release (azurelinux) - uses: softprops/action-gh-release@v1 - with: - name: Release ${{ github.event.inputs.versionName }} - tag_name: ${{ github.event.inputs.versionName }} - target_commitish: ${{ github.sha }} - files: | - ${{ github.workspace }}/azurelinux/root/rpmbuild/RPMS/x86_64/aznfs-azurelinux-${{ github.event.inputs.versionName }}-1.x86_64.rpm - body: | - New Release ${{ github.event.inputs.versionName }} + # - name: Create Release (azurelinux) + # uses: softprops/action-gh-release@v1 + # with: + # name: Release ${{ github.event.inputs.versionName }} + # tag_name: ${{ github.event.inputs.versionName }} + # target_commitish: ${{ github.sha }} + # files: | + # ${{ github.workspace }}/azurelinux/root/rpmbuild/RPMS/x86_64/aznfs-azurelinux-${{ github.event.inputs.versionName }}-1.x86_64.rpm + # body: | + # New Release ${{ github.event.inputs.versionName }} package_mount_helper_arm64: name: Package and Release ${{ github.event.inputs.versionName }} for arm64 (Test Release) From caf4fc331d6c4ee013685a4c0a8c2bdb3159a580 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Wed, 1 Oct 2025 04:33:11 +0530 Subject: [PATCH 07/21] Initial changes for azurelinux pipeline setup devops2 --- .github/workflows/prerelease.yml | 75 ++++++++++++++++---------------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 5d88b0f91..2cef83493 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -75,47 +75,46 @@ jobs: body: | New Test Release ${{ github.event.inputs.versionName }} - # NOTE: UNCOMMENT THIS JOB ONCE AZURE LINUX RUNNER IS CONNECTED - # - # package_mount_helper_azurelinux: - # name: Package and Release ${{ github.event.inputs.versionName }} for azurelinux - # runs-on: self-hosted - # steps: - # - name: Checkout repository - # uses: actions/checkout@v3 - # with: - # path: azurelinux + # NOTE: COMMENT THIS JOB IF AZURE LINUX RUNNER IS NOT CONNECTED + package_mount_helper_azurelinux: + name: Package and Release ${{ github.event.inputs.versionName }} for azurelinux + runs-on: self-hosted + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + path: azurelinux - # - name: Install prerequisites on Azure Linux - # run: | - # sudo tdnf update -y - # sudo tdnf install -y glibc-static rpm-build perl perl-IPC-Cmd \ - # gcc gcc-c++ make git curl unzip zip tar cmake \ - # zlib zlib-devel libuuid-devel gnutls gnutls-devel nfs-utils \ - # libyaml-devel spdlog-devel nlohmann-json-devel cmake make \ - # pkgconf pkgconf-m4 pkgconf-pkg-config ninja-build binutils \ - # kernel-headers kernel-devel autoconf automake libtool + - name: Install prerequisites on Azure Linux + run: | + sudo tdnf update -y + sudo tdnf install -y glibc-static rpm-build perl perl-IPC-Cmd \ + gcc gcc-c++ make git curl unzip zip tar cmake \ + zlib zlib-devel libuuid-devel gnutls gnutls-devel nfs-utils \ + libyaml-devel spdlog-devel nlohmann-json-devel cmake make \ + pkgconf pkgconf-m4 pkgconf-pkg-config ninja-build binutils \ + kernel-headers kernel-devel autoconf automake libtool - # - name: Run Package.sh for azurelinux - # run: | - # export RELEASE_NUMBER=${{ github.event.inputs.versionName }} - # export STG_DIR=$GITHUB_WORKSPACE - # export SOURCE_DIR=$GITHUB_WORKSPACE/azurelinux - # export BUILD_TYPE=${{ github.event.inputs.buildType }} - # export BUILD_MACHINE=azurelinux - # chmod +x $SOURCE_DIR/package.sh - # $SOURCE_DIR/package.sh + - name: Run Package.sh for azurelinux + run: | + export RELEASE_NUMBER=${{ github.event.inputs.versionName }} + export STG_DIR=$GITHUB_WORKSPACE + export SOURCE_DIR=$GITHUB_WORKSPACE/azurelinux + export BUILD_TYPE=${{ github.event.inputs.buildType }} + export BUILD_MACHINE=azurelinux + chmod +x $SOURCE_DIR/package.sh + $SOURCE_DIR/package.sh - # - name: Create Release (azurelinux) - # uses: softprops/action-gh-release@v1 - # with: - # name: Release ${{ github.event.inputs.versionName }} - # tag_name: ${{ github.event.inputs.versionName }} - # target_commitish: ${{ github.sha }} - # files: | - # ${{ github.workspace }}/azurelinux/root/rpmbuild/RPMS/x86_64/aznfs-azurelinux-${{ github.event.inputs.versionName }}-1.x86_64.rpm - # body: | - # New Release ${{ github.event.inputs.versionName }} + - name: Create Release (azurelinux) + uses: softprops/action-gh-release@v1 + with: + name: Release ${{ github.event.inputs.versionName }} + tag_name: ${{ github.event.inputs.versionName }} + target_commitish: ${{ github.sha }} + files: | + ${{ github.workspace }}/azurelinux/root/rpmbuild/RPMS/x86_64/aznfs-azurelinux-${{ github.event.inputs.versionName }}-1.x86_64.rpm + body: | + New Release ${{ github.event.inputs.versionName }} package_mount_helper_arm64: name: Package and Release ${{ github.event.inputs.versionName }} for arm64 (Test Release) From 18a505ff39ba8ca81be8d2f5751183c86a3931d1 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Mon, 6 Oct 2025 12:09:06 +0530 Subject: [PATCH 08/21] Initial changes for azurelinux pipeline setup devops3-fuse --- .github/workflows/prerelease.yml | 6 +- packaging/aznfs/RPM/aznfs.spec | 2 +- turbonfs/CMakeLists.txt | 145 +++++++++++++++++++------------ 3 files changed, 94 insertions(+), 59 deletions(-) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 2cef83493..c95aabc5c 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -27,7 +27,7 @@ permissions: contents: write jobs: - package_mount_helper_amd64: + package_mount_helper_ubuntu_amd64: name: Package and Release ${{ github.event.inputs.versionName }} for amd64 (Test Release) runs-on: ubuntu-22.04 steps: @@ -76,7 +76,7 @@ jobs: New Test Release ${{ github.event.inputs.versionName }} # NOTE: COMMENT THIS JOB IF AZURE LINUX RUNNER IS NOT CONNECTED - package_mount_helper_azurelinux: + package_mount_helper_azurelinux_amd64: name: Package and Release ${{ github.event.inputs.versionName }} for azurelinux runs-on: self-hosted steps: @@ -116,7 +116,7 @@ jobs: body: | New Release ${{ github.event.inputs.versionName }} - package_mount_helper_arm64: + package_mount_helper_ubuntu_arm64: name: Package and Release ${{ github.event.inputs.versionName }} for arm64 (Test Release) runs-on: ubuntu-22.04-arm steps: diff --git a/packaging/aznfs/RPM/aznfs.spec b/packaging/aznfs/RPM/aznfs.spec index 11f086b3f..d847e5316 100644 --- a/packaging/aznfs/RPM/aznfs.spec +++ b/packaging/aznfs/RPM/aznfs.spec @@ -9,7 +9,7 @@ Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, ipro Recommends: build-essential %elif 0%{?azurelinux_build} -Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, stunnel, net-tools, gnutls, jemalloc, libasan +Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, stunnel, net-tools, gnutls, jemalloc, libasan, fuse3 %else Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, stunnel, net-tools diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index ab94a6828..0ea30fc3e 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -27,7 +27,6 @@ option(ENABLE_JEMALLOC "Use jemalloc for malloc/free/new/delete" ON) option(ENABLE_INSECURE_AUTH_FOR_DEVTEST "Enable AZAUTH for non-TLS connections" OFF) option(ENABLE_DYNAMIC_LINKS "Use dynamic linking instead of bundling libs" OFF) -message(STATUS "ENABLE_INSECURE_AUTH_FOR_DEVTEST = ${ENABLE_INSECURE_AUTH_FOR_DEVTEST}") # # Builds that make it to customers need to be extra careful about any unnecessary # logging. Some warning logs we have in our code are to attract developer @@ -87,8 +86,11 @@ endif() set(INSTALL_BIN_DIR "${CMAKE_INSTALL_PREFIX}/bin" CACHE PATH "Installation directory for binaries") set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH}) -set(LIBFUSE_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/extern/libfuse") -set(LIBFUSE_BUILD_DIR "${CMAKE_BINARY_DIR}/extern/libfuse") +# Only set libfuse paths for static builds +if(NOT ENABLE_DYNAMIC_LINKS) + set(LIBFUSE_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/extern/libfuse") + set(LIBFUSE_BUILD_DIR "${CMAKE_BINARY_DIR}/extern/libfuse") +endif() # # We add the following submodule(s), update them as needed, before starting. @@ -125,8 +127,11 @@ if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/spdlog/CMakeLists.txt") message(FATAL_ERROR "The spdlog submodule was not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.") endif() -if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/libfuse/meson.build") - message(FATAL_ERROR "The libfuse submodule was not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.") +# Only check libfuse submodule for static builds +if(NOT ENABLE_DYNAMIC_LINKS) + if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/libfuse/meson.build") + message(FATAL_ERROR "The libfuse submodule was not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.") + endif() endif() if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/vcpkg/bootstrap-vcpkg.sh") @@ -172,59 +177,71 @@ if(NOT PKG_CONFIG_FOUND) endif() # -# We need meson to compile libfuse. +# Handle FUSE3 based on build type # -find_program(MESON_EXECUTABLE meson QUIET) -if(NOT MESON_FOUND) - message(STATUS "meson not found, trying to install meson!") - execute_process(COMMAND sudo ${PKG_MGR} install -y meson - RESULT_VARIABLE MESON_INSTALL_RESULT) - if(NOT MESON_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "${PKG_MGR} install meson failed with ${MESON_INSTALL_RESULT}, try installing meson manually and then run cmake again") - else() - find_program(MESON_EXECUTABLE meson REQUIRED) +if(ENABLE_DYNAMIC_LINKS) + # + # For dynamic builds, use system-installed fuse3 shared library + # + message(STATUS "Using system-installed fuse3 (dynamic linking)") + pkg_search_module(FUSE3 REQUIRED fuse3) + + if(NOT FUSE3_FOUND) + message(FATAL_ERROR "fuse3 not found! Please install fuse3-devel package: sudo tdnf install -y fuse3-devel") + endif() + + message(STATUS "Found fuse3 libraries: ${FUSE3_LIBRARIES}") + message(STATUS "Found fuse3 include dirs: ${FUSE3_INCLUDE_DIRS}") + message(STATUS "Found fuse3 library dirs: ${FUSE3_LIBRARY_DIRS}") +else() + # + # For static builds, build libfuse from source + # + message(STATUS "Building libfuse from source (static linking)") + + # + # We need meson to compile libfuse. + # + find_program(MESON_EXECUTABLE meson QUIET) + if(NOT MESON_FOUND) + message(STATUS "meson not found, trying to install meson!") + execute_process(COMMAND sudo ${PKG_MGR} install -y meson + RESULT_VARIABLE MESON_INSTALL_RESULT) + if(NOT MESON_INSTALL_RESULT EQUAL "0") + message(FATAL_ERROR "${PKG_MGR} install meson failed with ${MESON_INSTALL_RESULT}, try installing meson manually and then run cmake again") + else() + find_program(MESON_EXECUTABLE meson REQUIRED) + endif() endif() -endif() -# -# We need ninja to install libfuse in build dir. -# -find_program(NINJA_EXECUTABLE ninja QUIET) -if(NOT NINJA_FOUND) - message(STATUS "ninja not found, trying to install ninja-build!") - execute_process(COMMAND sudo ${PKG_MGR} install -y ninja-build - RESULT_VARIABLE NINJA_INSTALL_RESULT) - if(NOT NINJA_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "${PKG_MGR} install ninja-build failed with ${NINJA_INSTALL_RESULT}, try installing ninja-build manually and then run cmake again") - else() - find_program(NINJA_EXECUTABLE ninja REQUIRED) + # + # We need ninja to install libfuse in build dir. + # + find_program(NINJA_EXECUTABLE ninja QUIET) + if(NOT NINJA_FOUND) + message(STATUS "ninja not found, trying to install ninja-build!") + execute_process(COMMAND sudo ${PKG_MGR} install -y ninja-build + RESULT_VARIABLE NINJA_INSTALL_RESULT) + if(NOT NINJA_INSTALL_RESULT EQUAL "0") + message(FATAL_ERROR "${PKG_MGR} install ninja-build failed with ${NINJA_INSTALL_RESULT}, try installing ninja-build manually and then run cmake again") + else() + find_program(NINJA_EXECUTABLE ninja REQUIRED) + endif() endif() -endif() -# Decide fuse3 paths based on ENABLE_DYNAMIC_LINKS. -if(ENABLE_DYNAMIC_LINKS) - # Dynamic: Meson installs directly to /usr/local/lib. - set(fuse3_LIBRARY "/usr/local/lib/libfuse3.a") -else() - # Static: Use multiarch layout. + # Use multiarch layout for static builds set(fuse3_LIBRARY "/usr/local/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/libfuse3.a") -endif() - -# Include path remains the same for both cases. -set(fuse3_INCLUDE_DIR "/usr/local/include") + set(fuse3_INCLUDE_DIR "/usr/local/include") -# Meson will install the fuse library and header files in the following paths. -# set(fuse3_LIBRARY "/usr/local/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/libfuse3.a") -# set(fuse3_INCLUDE_DIR "/usr/local/include") + add_custom_command( + OUTPUT ${fuse3_LIBRARY} + COMMAND ${MESON_EXECUTABLE} setup ${LIBFUSE_BUILD_DIR} ${LIBFUSE_SOURCE_DIR} --default-library=static --buildtype=${MESON_BUILD_TYPE} + COMMAND ${MESON_EXECUTABLE} compile -C ${LIBFUSE_BUILD_DIR} + COMMAND sudo ${MESON_EXECUTABLE} install -C ${LIBFUSE_BUILD_DIR} + ) -add_custom_command( - OUTPUT ${fuse3_LIBRARY} - COMMAND ${MESON_EXECUTABLE} setup ${LIBFUSE_BUILD_DIR} ${LIBFUSE_SOURCE_DIR} --default-library=static --buildtype=${MESON_BUILD_TYPE} - COMMAND ${MESON_EXECUTABLE} compile -C ${LIBFUSE_BUILD_DIR} - COMMAND sudo ${MESON_EXECUTABLE} install -C ${LIBFUSE_BUILD_DIR} -) - -add_custom_target(libfuse ALL DEPENDS ${fuse3_LIBRARY}) + add_custom_target(libfuse ALL DEPENDS ${fuse3_LIBRARY}) +endif() # # We need GnuTLS for secure libnfs transport. @@ -401,7 +418,10 @@ else() set(sources ${sources} src/main.cpp) add_executable(${CMAKE_PROJECT_NAME} ${sources}) -add_dependencies(${CMAKE_PROJECT_NAME} libfuse) + # Only add libfuse dependency for static builds + if(NOT ENABLE_DYNAMIC_LINKS) + add_dependencies(${CMAKE_PROJECT_NAME} libfuse) + endif() endif() # All include directories. @@ -416,10 +436,15 @@ target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE "${PROJECT_SOURCE_DIR}/extern/spdlog/include") if(NOT ENABLE_NO_FUSE) -target_include_directories(${CMAKE_PROJECT_NAME} - PRIVATE "${fuse3_INCLUDE_DIR}") -target_link_libraries(${CMAKE_PROJECT_NAME} - ${fuse3_LIBRARY}) + if(ENABLE_DYNAMIC_LINKS) + # Use system fuse3 include directories + target_include_directories(${CMAKE_PROJECT_NAME} + PRIVATE ${FUSE3_INCLUDE_DIRS}) + else() + # Use built-from-source fuse3 include directory + target_include_directories(${CMAKE_PROJECT_NAME} + PRIVATE "${fuse3_INCLUDE_DIR}") + endif() endif() target_compile_options(${CMAKE_PROJECT_NAME} @@ -490,6 +515,11 @@ if(ENABLE_DYNAMIC_LINKS) nlohmann_json::nlohmann_json uuid GnuTLS::GnuTLS) + + # Link against system fuse3 (only if not building nofuse) + if(NOT ENABLE_NO_FUSE) + target_link_libraries(${CMAKE_PROJECT_NAME} ${FUSE3_LIBRARIES}) + endif() else() # Static linking: Use static-libgcc/libstdc++ and explicit gnutls libraries target_link_libraries(${CMAKE_PROJECT_NAME} @@ -506,6 +536,11 @@ else() Azure::azure-storage-blobs nlohmann_json::nlohmann_json uuid) + + # Link against static fuse3 (only if not building nofuse) + if(NOT ENABLE_NO_FUSE) + target_link_libraries(${CMAKE_PROJECT_NAME} ${fuse3_LIBRARY}) + endif() endif() if(ENABLE_TCMALLOC) From 525afded2deb7a97c0f21461ed767af4c36ff7d8 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Mon, 6 Oct 2025 12:49:38 +0530 Subject: [PATCH 09/21] Debug logs added --- package.sh | 21 +++++++++++++++++++++ turbonfs/CMakeLists.txt | 12 ++++++++++++ 2 files changed, 33 insertions(+) diff --git a/package.sh b/package.sh index 44cf82468..b101759c9 100755 --- a/package.sh +++ b/package.sh @@ -294,6 +294,27 @@ cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DPACKAGE_VERSION="${RELEASE_NUMBER}" \ -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake .. make + +# ****** TODO: REMOVE LATER ************** +# Verify the built binary immediately after make +echo "===== POST-BUILD VERIFICATION =====" +BUILT_BINARY="${SOURCE_DIR}/turbonfs/build/aznfsclient" +echo "Checking: ${BUILT_BINARY}" +ls -lh "${BUILT_BINARY}" +file "${BUILT_BINARY}" +echo "LDD output:" +ldd "${BUILT_BINARY}" | grep -E "fuse3|libfuse|libasan" + +if [ "$BUILD_MACHINE" == "azurelinux" ]; then + if ldd "${BUILT_BINARY}" | grep -q "libfuse3.so"; then + echo "✓ fuse3 dynamically linked" + else + echo "✗ ERROR: fuse3 NOT dynamically linked!" + exit 1 + fi +fi +echo "=====================================" + popd if [ "$BUILD_MACHINE" != "azurelinux" ]; then diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index 0ea30fc3e..4fa0ec7fd 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -567,3 +567,15 @@ endforeach() message(STATUS "===============================================") install(TARGETS ${CMAKE_PROJECT_NAME}) + +# *********** TODO: REMOVE LATER. ********* +# Add post-build verification for dynamic builds +if(ENABLE_DYNAMIC_LINKS AND NOT ENABLE_NO_FUSE) + add_custom_command(TARGET ${CMAKE_PROJECT_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E echo "===== Verifying dynamic fuse3 linking =====" + COMMAND ldd $ | grep -E "(fuse3|libfuse)" || + (echo "ERROR: fuse3 NOT found in ldd output!" && exit 1) + COMMAND ${CMAKE_COMMAND} -E echo "===== fuse3 successfully linked =====" + COMMENT "Verifying fuse3 dynamic linking" + ) +endif() \ No newline at end of file From 1479a0e8b5a2f130ffb756696e0237b1079f1f5b Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Mon, 6 Oct 2025 13:05:25 +0530 Subject: [PATCH 10/21] Debug logs added --- turbonfs/CMakeLists.txt | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index 4fa0ec7fd..0ea30fc3e 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -567,15 +567,3 @@ endforeach() message(STATUS "===============================================") install(TARGETS ${CMAKE_PROJECT_NAME}) - -# *********** TODO: REMOVE LATER. ********* -# Add post-build verification for dynamic builds -if(ENABLE_DYNAMIC_LINKS AND NOT ENABLE_NO_FUSE) - add_custom_command(TARGET ${CMAKE_PROJECT_NAME} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E echo "===== Verifying dynamic fuse3 linking =====" - COMMAND ldd $ | grep -E "(fuse3|libfuse)" || - (echo "ERROR: fuse3 NOT found in ldd output!" && exit 1) - COMMAND ${CMAKE_COMMAND} -E echo "===== fuse3 successfully linked =====" - COMMENT "Verifying fuse3 dynamic linking" - ) -endif() \ No newline at end of file From c6ccd3483c737e5a394ad83fe1e17a8abd24db3a Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Mon, 6 Oct 2025 13:12:38 +0530 Subject: [PATCH 11/21] Debug logs added --- package.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package.sh b/package.sh index b101759c9..61bf43b6a 100755 --- a/package.sh +++ b/package.sh @@ -303,14 +303,13 @@ echo "Checking: ${BUILT_BINARY}" ls -lh "${BUILT_BINARY}" file "${BUILT_BINARY}" echo "LDD output:" -ldd "${BUILT_BINARY}" | grep -E "fuse3|libfuse|libasan" +ldd "${BUILT_BINARY}" if [ "$BUILD_MACHINE" == "azurelinux" ]; then if ldd "${BUILT_BINARY}" | grep -q "libfuse3.so"; then echo "✓ fuse3 dynamically linked" else echo "✗ ERROR: fuse3 NOT dynamically linked!" - exit 1 fi fi echo "=====================================" From df25ab611737686e5247bd92c097155c2aff0043 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Mon, 6 Oct 2025 14:27:33 +0530 Subject: [PATCH 12/21] Debug logs added and update fuse3-devel --- turbonfs/CMakeLists.txt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index 0ea30fc3e..78170dd8e 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -184,7 +184,7 @@ if(ENABLE_DYNAMIC_LINKS) # For dynamic builds, use system-installed fuse3 shared library # message(STATUS "Using system-installed fuse3 (dynamic linking)") - pkg_search_module(FUSE3 REQUIRED fuse3) + pkg_search_module(FUSE3 REQUIRED fuse3-devel) if(NOT FUSE3_FOUND) message(FATAL_ERROR "fuse3 not found! Please install fuse3-devel package: sudo tdnf install -y fuse3-devel") @@ -193,6 +193,15 @@ if(ENABLE_DYNAMIC_LINKS) message(STATUS "Found fuse3 libraries: ${FUSE3_LIBRARIES}") message(STATUS "Found fuse3 include dirs: ${FUSE3_INCLUDE_DIRS}") message(STATUS "Found fuse3 library dirs: ${FUSE3_LIBRARY_DIRS}") + + target_include_directories(${CMAKE_PROJECT_NAME} + PRIVATE + ${FUSE3_INCLUDE_DIRS} + ) + + target_link_libraries(${CMAKE_PROJECT_NAME} + ${FUSE3_LIBRARIES} + ) else() # # For static builds, build libfuse from source From 2acbfa9299bc87e7039757a4915e61903500399e Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Mon, 6 Oct 2025 14:36:15 +0530 Subject: [PATCH 13/21] Debug logs added and update fuse3-devel --- turbonfs/CMakeLists.txt | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index 78170dd8e..c5c6a85a6 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -184,24 +184,40 @@ if(ENABLE_DYNAMIC_LINKS) # For dynamic builds, use system-installed fuse3 shared library # message(STATUS "Using system-installed fuse3 (dynamic linking)") - pkg_search_module(FUSE3 REQUIRED fuse3-devel) - + + # First try to find fuse3 via pkg-config + find_package(PkgConfig REQUIRED) + pkg_search_module(FUSE3 QUIET fuse3) + if(NOT FUSE3_FOUND) - message(FATAL_ERROR "fuse3 not found! Please install fuse3-devel package: sudo tdnf install -y fuse3-devel") + message(STATUS "fuse3 not found via pkg-config, trying to install the devel package") + + if(PKG_MGR STREQUAL "apt") + set(FUSE3_PKG "libfuse3-dev") + else() + set(FUSE3_PKG "fuse3-devel") + endif() + + execute_process(COMMAND sudo ${PKG_MGR} install -y ${FUSE3_PKG} + RESULT_VARIABLE FUSE3_INSTALL_RESULT) + + if(NOT FUSE3_INSTALL_RESULT EQUAL "0") + message(FATAL_ERROR "Failed to install ${FUSE3_PKG}, please install manually") + endif() + + # Retry finding fuse3 after installation + pkg_search_module(FUSE3 REQUIRED fuse3) endif() - + message(STATUS "Found fuse3 libraries: ${FUSE3_LIBRARIES}") message(STATUS "Found fuse3 include dirs: ${FUSE3_INCLUDE_DIRS}") message(STATUS "Found fuse3 library dirs: ${FUSE3_LIBRARY_DIRS}") target_include_directories(${CMAKE_PROJECT_NAME} - PRIVATE - ${FUSE3_INCLUDE_DIRS} + PRIVATE ${FUSE3_INCLUDE_DIRS} ) - target_link_libraries(${CMAKE_PROJECT_NAME} - ${FUSE3_LIBRARIES} - ) + target_link_libraries(${CMAKE_PROJECT_NAME} ${FUSE3_LIBRARIES}) else() # # For static builds, build libfuse from source From f96fdee6af6041e124a3fe168e4c913643420808 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Mon, 6 Oct 2025 14:39:14 +0530 Subject: [PATCH 14/21] Debug logs added and update fuse3-devel --- .github/workflows/prerelease.yml | 2 +- turbonfs/CMakeLists.txt | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index c95aabc5c..1d48336bb 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -93,7 +93,7 @@ jobs: zlib zlib-devel libuuid-devel gnutls gnutls-devel nfs-utils \ libyaml-devel spdlog-devel nlohmann-json-devel cmake make \ pkgconf pkgconf-m4 pkgconf-pkg-config ninja-build binutils \ - kernel-headers kernel-devel autoconf automake libtool + kernel-headers kernel-devel autoconf automake libtool fuse3-devel - name: Run Package.sh for azurelinux run: | diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index c5c6a85a6..d0d7cf360 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -212,12 +212,6 @@ if(ENABLE_DYNAMIC_LINKS) message(STATUS "Found fuse3 libraries: ${FUSE3_LIBRARIES}") message(STATUS "Found fuse3 include dirs: ${FUSE3_INCLUDE_DIRS}") message(STATUS "Found fuse3 library dirs: ${FUSE3_LIBRARY_DIRS}") - - target_include_directories(${CMAKE_PROJECT_NAME} - PRIVATE ${FUSE3_INCLUDE_DIRS} - ) - - target_link_libraries(${CMAKE_PROJECT_NAME} ${FUSE3_LIBRARIES}) else() # # For static builds, build libfuse from source From 5d3967487c1fb4a98aee978fde0248953d5485a1 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Tue, 7 Oct 2025 11:20:59 +0530 Subject: [PATCH 15/21] update review --- generate_package.sh | 27 ++++++++++++--------------- package.sh | 20 -------------------- turbonfs/CMakeLists.txt | 8 -------- 3 files changed, 12 insertions(+), 43 deletions(-) diff --git a/generate_package.sh b/generate_package.sh index a8bbc7c6f..ff7a63d7c 100644 --- a/generate_package.sh +++ b/generate_package.sh @@ -67,21 +67,18 @@ generate_rpm_package() # copy the aznfsclient config file. cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ - if [ "$rpm_dir" != "azurelinux" ]; then - # copy the aznfsclient binary. - cp -avf ${aznfsclient} ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient - else - # Define the final target location for aznfsclient - aznfsclient_target="${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient" - - # Copy the built aznfsclient binary to the target - cp -avf "${SOURCE_DIR}/turbonfs/build/aznfsclient" "${aznfsclient_target}" - - # Optional: fail early if copy fails - if [ ! -f "${aznfsclient_target}" ]; then - echo "Error: aznfsclient failed to copy to ${aznfsclient_target}" - exit 1 - fi + # Common handling for aznfsclient binary copy across all distros. + # Define the source and destination for aznfsclient. + aznfsclient_src="${SOURCE_DIR}/turbonfs/build/aznfsclient" + aznfsclient_target="${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient" + + # Copy the built aznfsclient binary to the target + cp -avf "${aznfsclient_src}" "${aznfsclient_target}" + + # Fail early if copy fails + if [ ! -f "${aznfsclient_target}" ]; then + echo "Error: aznfsclient failed to copy to ${aznfsclient_target}" + exit 1 fi if [ "$rpm_dir" != "azurelinux" ]; then diff --git a/package.sh b/package.sh index 61bf43b6a..44cf82468 100755 --- a/package.sh +++ b/package.sh @@ -294,26 +294,6 @@ cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DPACKAGE_VERSION="${RELEASE_NUMBER}" \ -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake .. make - -# ****** TODO: REMOVE LATER ************** -# Verify the built binary immediately after make -echo "===== POST-BUILD VERIFICATION =====" -BUILT_BINARY="${SOURCE_DIR}/turbonfs/build/aznfsclient" -echo "Checking: ${BUILT_BINARY}" -ls -lh "${BUILT_BINARY}" -file "${BUILT_BINARY}" -echo "LDD output:" -ldd "${BUILT_BINARY}" - -if [ "$BUILD_MACHINE" == "azurelinux" ]; then - if ldd "${BUILT_BINARY}" | grep -q "libfuse3.so"; then - echo "✓ fuse3 dynamically linked" - else - echo "✗ ERROR: fuse3 NOT dynamically linked!" - fi -fi -echo "=====================================" - popd if [ "$BUILD_MACHINE" != "azurelinux" ]; then diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index d0d7cf360..aa308f724 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -577,12 +577,4 @@ if(ENABLE_JEMALLOC) endif() endif() -# --- Debug: Print all static libraries linked --- -get_target_property(LINK_LIBS ${CMAKE_PROJECT_NAME} LINK_LIBRARIES) -message(STATUS "==== Libraries linked to ${CMAKE_PROJECT_NAME} ====") -foreach(lib IN LISTS LINK_LIBS) - message(STATUS " ${lib}") -endforeach() -message(STATUS "===============================================") - install(TARGETS ${CMAKE_PROJECT_NAME}) From 0062819a9aac0c966c97e12dfea9d350a4a68a25 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Tue, 7 Oct 2025 11:35:29 +0530 Subject: [PATCH 16/21] update --- .github/workflows/prerelease.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 1d48336bb..45d647e9a 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -29,7 +29,7 @@ permissions: jobs: package_mount_helper_ubuntu_amd64: name: Package and Release ${{ github.event.inputs.versionName }} for amd64 (Test Release) - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Check Version Number shell: bash @@ -118,7 +118,7 @@ jobs: package_mount_helper_ubuntu_arm64: name: Package and Release ${{ github.event.inputs.versionName }} for arm64 (Test Release) - runs-on: ubuntu-22.04-arm + runs-on: ubuntu-24.04-arm steps: - name: Check Version Number shell: bash From af8083f25d78e30ddff385c2cfc3a39be70975ee Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Tue, 7 Oct 2025 11:48:09 +0530 Subject: [PATCH 17/21] update --- .github/workflows/prerelease.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 45d647e9a..1d48336bb 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -29,7 +29,7 @@ permissions: jobs: package_mount_helper_ubuntu_amd64: name: Package and Release ${{ github.event.inputs.versionName }} for amd64 (Test Release) - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 steps: - name: Check Version Number shell: bash @@ -118,7 +118,7 @@ jobs: package_mount_helper_ubuntu_arm64: name: Package and Release ${{ github.event.inputs.versionName }} for arm64 (Test Release) - runs-on: ubuntu-24.04-arm + runs-on: ubuntu-22.04-arm steps: - name: Check Version Number shell: bash From 3e1282945ae9e6b0ffddb50dfe66b2d6c3defb5a Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Fri, 10 Oct 2025 11:29:05 +0530 Subject: [PATCH 18/21] Update dependency --- .github/aznfs-build.yaml | 2 +- .github/workflows/prerelease.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/aznfs-build.yaml b/.github/aznfs-build.yaml index ae0eb3e17..8e55ac15f 100644 --- a/.github/aznfs-build.yaml +++ b/.github/aznfs-build.yaml @@ -98,7 +98,7 @@ stages: zlib zlib-devel libuuid-devel gnutls gnutls-devel nfs-utils \ libyaml-devel spdlog-devel nlohmann-json-devel cmake make \ pkgconf pkgconf-m4 pkgconf-pkg-config ninja-build binutils \ - kernel-headers kernel-devel autoconf automake libtool jemalloc-devel + kernel-headers kernel-devel autoconf automake libtool jemalloc-devel fuse3-devel displayName: "Install prerequisites on Azure Linux" # note: remove later. diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 1d48336bb..47fcd2703 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -93,7 +93,7 @@ jobs: zlib zlib-devel libuuid-devel gnutls gnutls-devel nfs-utils \ libyaml-devel spdlog-devel nlohmann-json-devel cmake make \ pkgconf pkgconf-m4 pkgconf-pkg-config ninja-build binutils \ - kernel-headers kernel-devel autoconf automake libtool fuse3-devel + kernel-headers kernel-devel autoconf automake libtool jemalloc-devel fuse3-devel - name: Run Package.sh for azurelinux run: | From 0d054573b65c2e409cbdece82100138913c51d56 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Tue, 28 Oct 2025 11:10:36 +0530 Subject: [PATCH 19/21] Updating w/o ubuntu and addressing review comments --- generate_package.sh | 86 ++++++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/generate_package.sh b/generate_package.sh index ff7a63d7c..ad00b62ba 100644 --- a/generate_package.sh +++ b/generate_package.sh @@ -67,19 +67,9 @@ generate_rpm_package() # copy the aznfsclient config file. cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ - # Common handling for aznfsclient binary copy across all distros. - # Define the source and destination for aznfsclient. - aznfsclient_src="${SOURCE_DIR}/turbonfs/build/aznfsclient" - aznfsclient_target="${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient" - - # Copy the built aznfsclient binary to the target - cp -avf "${aznfsclient_src}" "${aznfsclient_target}" - - # Fail early if copy fails - if [ ! -f "${aznfsclient_target}" ]; then - echo "Error: aznfsclient failed to copy to ${aznfsclient_target}" - exit 1 - fi + # aznfsclient in the final target dir. + aznfsclient=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient + cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} if [ "$rpm_dir" != "azurelinux" ]; then # @@ -122,9 +112,9 @@ generate_rpm_package() sed -i -e "s/PROCPS_PACKAGE_NAME/sysvinit-tools/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec sed -i -e "s/DISTRO/suse/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec else - if [ "$rpm_dir" == "azurelinux" ]; then - sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}-azurelinux/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec - fi + # if [ "$rpm_dir" == "azurelinux" ]; then + # sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}-azurelinux/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + # fi sed -i -e "s/NETCAT_PACKAGE_NAME/nmap-ncat/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec # In new versions of Centos/RedHat/Rocky, procps-ng provides pidof. For older versions, it is provided by sysvinit-tools but since it is not @@ -155,31 +145,31 @@ rpm_buildroot_dir="${rpmbuild_dir}/BUILDROOT" sed -i -e "s/RELEASE_NUMBER=x.y.z/RELEASE_NUMBER=${RELEASE_NUMBER}/g" ${SOURCE_DIR}/scripts/aznfs_install.sh -if [ "$BUILD_MACHINE" != "azurelinux" ]; then - ######################### - # Generate .deb package # - ######################### +# if [ "$BUILD_MACHINE" != "azurelinux" ]; then +# ######################### +# # Generate .deb package # +# ######################### - # Create the directory to hold the package control and data files for deb package. - mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN +# # Create the directory to hold the package control and data files for deb package. +# mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN - # Copy the debian control file(s) and maintainer scripts. - cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ - chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* +# # Copy the debian control file(s) and maintainer scripts. +# cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ +# chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* - # Insert current release number. - sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control - sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control +# # Insert current release number. +# sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control +# sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control - # Copy other static package file(s). - mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin - cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ - cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ +# # Copy other static package file(s). +# mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin +# cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ +# cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ - # Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. - mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin - gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs -fi +# # Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. +# mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin +# gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs +# fi # @@ -223,6 +213,30 @@ make popd if [ "$BUILD_MACHINE" != "azurelinux" ]; then + ######################### + # Generate .deb package # + ######################### + + # Create the directory to hold the package control and data files for deb package. + mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN + + # Copy the debian control file(s) and maintainer scripts. + cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ + chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* + + # Insert current release number. + sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control + sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control + + # Copy other static package file(s). + mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin + cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ + cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ + + # Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. + mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin + gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs + mkdir -p ${STG_DIR}/deb/${pkg_dir}${opt_dir} cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ cp -avf ${SOURCE_DIR}/src/mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ From 8977ee29744ea4673f3fb9cb78abb61151d32a04 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Tue, 28 Oct 2025 11:28:41 +0530 Subject: [PATCH 20/21] update package.sh --- package.sh | 70 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/package.sh b/package.sh index 44cf82468..c2197e05f 100755 --- a/package.sh +++ b/package.sh @@ -82,12 +82,6 @@ generate_rpm_package() # Copy the built aznfsclient binary to the target cp -avf "${SOURCE_DIR}/turbonfs/build/aznfsclient" "${aznfsclient_target}" - - # Optional: fail early if copy fails - if [ ! -f "${aznfsclient_target}" ]; then - echo "Error: aznfsclient failed to copy to ${aznfsclient_target}" - exit 1 - fi fi if [ "$rpm_dir" != "azurelinux" ]; then @@ -230,31 +224,31 @@ rpm_buildroot_dir="${rpmbuild_dir}/BUILDROOT" # Insert release number to aznfs_install.sh sed -i -e "s/RELEASE_NUMBER=x.y.z/RELEASE_NUMBER=${RELEASE_NUMBER}/g" ${SOURCE_DIR}/scripts/aznfs_install.sh -if [ "$BUILD_MACHINE" != "azurelinux" ]; then - ######################### - # Generate .deb package # - ######################### +# if [ "$BUILD_MACHINE" != "azurelinux" ]; then +# ######################### +# # Generate .deb package # +# ######################### - # Create the directory to hold the package control and data files for deb package. - mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN +# # Create the directory to hold the package control and data files for deb package. +# mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN - # Copy the debian control file(s) and maintainer scripts. - cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ - chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* +# # Copy the debian control file(s) and maintainer scripts. +# cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ +# chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* - # Insert current release number. - sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control - sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control +# # Insert current release number. +# sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control +# sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control - # Copy other static package file(s). - mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin - cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ - cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ +# # Copy other static package file(s). +# mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin +# cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ +# cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ - # Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. - mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin - gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs -fi +# # Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. +# mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin +# gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs +# fi # # We build the turbonfs project here, note that we can set all cmake options in the @@ -297,6 +291,30 @@ make popd if [ "$BUILD_MACHINE" != "azurelinux" ]; then + ######################### + # Generate .deb package # + ######################### + + # Create the directory to hold the package control and data files for deb package. + mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN + + # Copy the debian control file(s) and maintainer scripts. + cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ + chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* + + # Insert current release number. + sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control + sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control + + # Copy other static package file(s). + mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin + cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ + cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ + + # Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. + mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin + gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs + mkdir -p ${STG_DIR}/deb/${pkg_dir}${opt_dir} cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ cp -avf ${SOURCE_DIR}/src/mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ From 4a31e5e3c9ed28751fec406552989f68e5c41a43 Mon Sep 17 00:00:00 2001 From: Palash Vij Date: Thu, 13 Nov 2025 10:31:50 +0530 Subject: [PATCH 21/21] update --- .github/aznfs-build.yaml | 19 +------------- generate_package.sh | 55 ++++++---------------------------------- package.sh | 32 ++++------------------- 3 files changed, 14 insertions(+), 92 deletions(-) diff --git a/.github/aznfs-build.yaml b/.github/aznfs-build.yaml index 8e55ac15f..2a8b2db39 100644 --- a/.github/aznfs-build.yaml +++ b/.github/aznfs-build.yaml @@ -80,7 +80,7 @@ stages: PathtoPublish: $(Build.ArtifactStagingDirectory) artifactName: 'aznfs-temp' - - job: package_aznfs_azurelinu_amd64 + - job: package_aznfs_azurelinux_amd64 displayName: Package and Release ${{ parameters.versionName }} for azurelinux pool: name: 'aznfsazurelinuxdemopool' # Use your self-hosted agent @@ -101,15 +101,6 @@ stages: kernel-headers kernel-devel autoconf automake libtool jemalloc-devel fuse3-devel displayName: "Install prerequisites on Azure Linux" - # note: remove later. - - script: | - echo "=== Validating Build Environment ===" - gcc --version - cmake --version - tdnf --version - echo "Architecture: $(uname -m)" - displayName: "Validate build environment" - - script: | export RELEASE_NUMBER=${{ parameters.versionName }} export STG_DIR=$(System.DefaultWorkingDirectory) @@ -120,14 +111,6 @@ stages: $SOURCE_DIR/generate_package.sh displayName: "Run Package Script" - # note: remove later. - - script: | - echo "=== Directory Structure ===" - find $(System.DefaultWorkingDirectory) -name "*.rpm" -type f - echo "=== STG_DIR Contents ===" - ls -la $(System.DefaultWorkingDirectory) - displayName: "Debug: List directory structure" - - script: | mkdir -p $(Build.ArtifactStagingDirectory)/artifacts/rpmazurelinux cp -f $(System.DefaultWorkingDirectory)/azurelinux/root/rpmbuild/RPMS/x86_64/aznfs-${{ parameters.versionName }}-1.x86_64.rpm $(Build.ArtifactStagingDirectory)/artifacts/rpmazurelinux diff --git a/generate_package.sh b/generate_package.sh index ad00b62ba..a1ae9f036 100644 --- a/generate_package.sh +++ b/generate_package.sh @@ -26,16 +26,9 @@ generate_rpm_package() custom_stunnel_required=0 azurelinux_build_required=0 - # Overwrite rpm_pkg_dir in case of RedHat7 and Centos7. - if [ "$rpm_dir" == "stunnel" ]; then - custom_stunnel_required=1 - fi - - # Overwrite rpm_pkg_dir in case of azurelinux. - if [ "$rpm_dir" == "azurelinux" ]; then - # rpm_pkg_dir="${pkg_name}-azurelinux-${RELEASE_NUMBER}-1.$arch" - azurelinux_build_required=1 - fi + # Set flags based on rpm_dir type + [ "$rpm_dir" == "stunnel" ] && custom_stunnel_required=1 + [ "$rpm_dir" == "azurelinux" ] && azurelinux_build_required=1 # Create the directory to hold the package spec and data files for RPM package. mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir} @@ -47,7 +40,11 @@ generate_rpm_package() # Compile mount.aznfs.c and put the executable into ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin. mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin - gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/mount.aznfs + if [ "$rpm_dir" == "azurelinux" ]; then + gcc ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/mount.aznfs + else + gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/mount.aznfs + fi mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir} cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ @@ -112,9 +109,6 @@ generate_rpm_package() sed -i -e "s/PROCPS_PACKAGE_NAME/sysvinit-tools/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec sed -i -e "s/DISTRO/suse/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec else - # if [ "$rpm_dir" == "azurelinux" ]; then - # sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}-azurelinux/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec - # fi sed -i -e "s/NETCAT_PACKAGE_NAME/nmap-ncat/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec # In new versions of Centos/RedHat/Rocky, procps-ng provides pidof. For older versions, it is provided by sysvinit-tools but since it is not @@ -144,39 +138,6 @@ rpm_buildroot_dir="${rpmbuild_dir}/BUILDROOT" # Insert release number to aznfs_install.sh sed -i -e "s/RELEASE_NUMBER=x.y.z/RELEASE_NUMBER=${RELEASE_NUMBER}/g" ${SOURCE_DIR}/scripts/aznfs_install.sh - -# if [ "$BUILD_MACHINE" != "azurelinux" ]; then -# ######################### -# # Generate .deb package # -# ######################### - -# # Create the directory to hold the package control and data files for deb package. -# mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN - -# # Copy the debian control file(s) and maintainer scripts. -# cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ -# chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* - -# # Insert current release number. -# sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control -# sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control - -# # Copy other static package file(s). -# mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin -# cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ -# cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ - -# # Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. -# mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin -# gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs -# fi - - -# -# We build the turbonfs project here, note that we can set all cmake options in the -# future using env variables. -# - pushd ${SOURCE_DIR}/turbonfs export VCPKG_ROOT=${SOURCE_DIR}/turbonfs/extern/vcpkg # We need to update the submodules before calling cmake as toolchain build expects it. diff --git a/package.sh b/package.sh index c2197e05f..8490327ad 100755 --- a/package.sh +++ b/package.sh @@ -53,7 +53,11 @@ generate_rpm_package() # Compile mount.aznfs.c and put the executable into ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin. mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin - gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/mount.aznfs + if [ "$rpm_dir" == "azurelinux" ]; then + gcc ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/mount.aznfs + else + gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/mount.aznfs + fi mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir} cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ @@ -224,32 +228,6 @@ rpm_buildroot_dir="${rpmbuild_dir}/BUILDROOT" # Insert release number to aznfs_install.sh sed -i -e "s/RELEASE_NUMBER=x.y.z/RELEASE_NUMBER=${RELEASE_NUMBER}/g" ${SOURCE_DIR}/scripts/aznfs_install.sh -# if [ "$BUILD_MACHINE" != "azurelinux" ]; then -# ######################### -# # Generate .deb package # -# ######################### - -# # Create the directory to hold the package control and data files for deb package. -# mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN - -# # Copy the debian control file(s) and maintainer scripts. -# cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ -# chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* - -# # Insert current release number. -# sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control -# sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control - -# # Copy other static package file(s). -# mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin -# cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ -# cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ - -# # Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. -# mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin -# gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs -# fi - # # We build the turbonfs project here, note that we can set all cmake options in the # future using env variables.