diff --git a/.github/aznfs-build.yaml b/.github/aznfs-build.yaml index 22ad311bd..71fd3eb82 100644 --- a/.github/aznfs-build.yaml +++ b/.github/aznfs-build.yaml @@ -78,6 +78,66 @@ stages: PathtoPublish: $(Build.ArtifactStagingDirectory) artifactName: 'aznfs-temp' + - job: package_aznfs_azurelinu_amd64 + displayName: Package and Release ${{ parameters.versionName }} for azurelinux + pool: + name: 'aznfsazurelinuxdemopool' # Use your self-hosted agent + + steps: + - checkout: self + path: azurelinux/AZNFS-mount + displayName: 'Checkout repository' + + - script: | + echo "Installing prerequisites on Azure Linux..." + sudo tdnf update -y + sudo tdnf install -y glibc-static rpm-build perl perl-IPC-Cmd \ + gcc gcc-c++ make git curl unzip zip tar cmake \ + zlib zlib-devel libuuid-devel gnutls gnutls-devel nfs-utils \ + libyaml-devel spdlog-devel nlohmann-json-devel cmake make \ + pkgconf pkgconf-m4 pkgconf-pkg-config ninja-build binutils \ + kernel-headers kernel-devel autoconf automake libtool jemalloc-devel + displayName: "Install prerequisites on Azure Linux" + + # note: remove later. + - script: | + echo "=== Validating Build Environment ===" + gcc --version + cmake --version + tdnf --version + echo "Architecture: $(uname -m)" + displayName: "Validate build environment" + + - script: | + export RELEASE_NUMBER=${{ parameters.versionName }} + export STG_DIR=$(System.DefaultWorkingDirectory) + export SOURCE_DIR=$(System.DefaultWorkingDirectory)/azurelinux + export BUILD_TYPE=${{ parameters.buildType }} + export BUILD_MACHINE=azurelinux + chmod +x $SOURCE_DIR/generate_package.sh + $SOURCE_DIR/generate_package.sh + displayName: "Run Package Script" + + # note: remove later. + - script: | + echo "=== Directory Structure ===" + find $(System.DefaultWorkingDirectory) -name "*.rpm" -type f + echo "=== STG_DIR Contents ===" + ls -la $(System.DefaultWorkingDirectory) + displayName: "Debug: List directory structure" + + - script: | + mkdir -p $(Build.ArtifactStagingDirectory)/artifacts/rpm + cp -f $(System.DefaultWorkingDirectory)/azurelinux/root/rpmbuild/RPMS/x86_64/aznfs-azurelinux-${{ parameters.versionName }}-1.x86_64.rpm $(Build.ArtifactStagingDirectory)/artifacts/rpm + echo "Listing Built Files..." + ls -R $(Build.ArtifactStagingDirectory) + displayName: "Copy RPM to Artifacts" + + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory) + artifactName: 'aznfs-temp' + - job: package_aznfs_arm64 displayName: Package and Release ${{ parameters.versionName }} for arm64 pool: diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index ba2da2b12..ce0508c9e 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -54,7 +54,8 @@ jobs: export RELEASE_NUMBER=${{ github.event.inputs.versionName }} export STG_DIR=$GITHUB_WORKSPACE/amd64 export SOURCE_DIR=$GITHUB_WORKSPACE/amd64 - export BUILD_TYPE=${{ github.events.inputs.buildType }} + export BUILD_TYPE=${{ github.event.inputs.buildType }} + export BUILD_MACHINE=ubuntu chmod +x $SOURCE_DIR/package.sh $SOURCE_DIR/package.sh - name: Create Test Release @@ -74,6 +75,46 @@ jobs: body: | New Test Release ${{ github.event.inputs.versionName }} + package_mount_helper_azurelinux: + name: Package and Release ${{ github.event.inputs.versionName }} for azurelinux + runs-on: self-hosted + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + path: azurelinux + + - name: Install prerequisites on Azure Linux + run: | + sudo tdnf update -y + sudo tdnf install -y glibc-static rpm-build perl perl-IPC-Cmd \ + gcc gcc-c++ make git curl unzip zip tar cmake \ + zlib zlib-devel libuuid-devel gnutls gnutls-devel nfs-utils \ + libyaml-devel spdlog-devel nlohmann-json-devel cmake make \ + pkgconf pkgconf-m4 pkgconf-pkg-config ninja-build binutils \ + kernel-headers kernel-devel autoconf automake libtool + + - name: Run Package.sh for azurelinux + run: | + export RELEASE_NUMBER=${{ github.event.inputs.versionName }} + export STG_DIR=$GITHUB_WORKSPACE + export SOURCE_DIR=$GITHUB_WORKSPACE/azurelinux + export BUILD_TYPE=${{ github.event.inputs.buildType }} + export BUILD_MACHINE=azurelinux + chmod +x $SOURCE_DIR/package.sh + $SOURCE_DIR/package.sh + + - name: Create Release (azurelinux) + uses: softprops/action-gh-release@v1 + with: + name: Release ${{ github.event.inputs.versionName }} + tag_name: ${{ github.event.inputs.versionName }} + target_commitish: ${{ github.sha }} + files: | + ${{ github.workspace }}/azurelinux/root/rpmbuild/RPMS/x86_64/aznfs-azurelinux-${{ github.event.inputs.versionName }}-1.x86_64.rpm + body: | + New Release ${{ github.event.inputs.versionName }} + package_mount_helper_arm64: name: Package and Release ${{ github.event.inputs.versionName }} for arm64 (Test Release) runs-on: ubuntu-22.04-arm @@ -101,7 +142,7 @@ jobs: export RELEASE_NUMBER=${{ github.event.inputs.versionName }} export STG_DIR=$GITHUB_WORKSPACE/arm64 export SOURCE_DIR=$GITHUB_WORKSPACE/arm64 - export BUILD_TYPE=${{ github.events.inputs.buildType }} + export BUILD_TYPE=${{ github.event.inputs.buildType }} chmod +x $SOURCE_DIR/package.sh $SOURCE_DIR/package.sh - name: Create Test Release diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3d4a592c1..c6c81ce19 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,6 +74,7 @@ jobs: export STG_DIR=$GITHUB_WORKSPACE/amd64 export SOURCE_DIR=$GITHUB_WORKSPACE/amd64 export BUILD_TYPE=${{ github.events.inputs.buildType }} + export BUILD_MACHINE=ubuntu chmod +x $SOURCE_DIR/package.sh $SOURCE_DIR/package.sh - name: Create Release @@ -137,3 +138,6 @@ jobs: ${{ github.workspace }}/arm64/tarball/aznfs-${{ github.event.inputs.versionName }}-1.aarch64.tar.gz body: | New Release ${{ github.event.inputs.versionName }} + + +# TODO: Add for arm64 too. \ No newline at end of file diff --git a/package.sh b/package.sh index 55eeb2524..44cf82468 100755 --- a/package.sh +++ b/package.sh @@ -24,12 +24,19 @@ generate_rpm_package() { rpm_dir=$1 custom_stunnel_required=0 + azurelinux_build_required=0 # Overwrite rpm_pkg_dir in case of SUSE. if [ "$rpm_dir" == "suse" ]; then rpm_pkg_dir="${pkg_name}_sles-${RELEASE_NUMBER}-1.$arch" fi + # Overwrite rpm_pkg_dir in case of azurelinux. + if [ "$rpm_dir" == "azurelinux" ]; then + rpm_pkg_dir="${pkg_name}-azurelinux-${RELEASE_NUMBER}-1.$arch" + azurelinux_build_required=1 + fi + # Overwrite rpm_pkg_dir in case of Mariner, RedHat7, and Centos7. if [ "$rpm_dir" == "stunnel" ]; then rpm_pkg_dir="${pkg_name}_stunnel_custom-${RELEASE_NUMBER}-1.$arch" @@ -66,17 +73,33 @@ generate_rpm_package() # copy the aznfsclient config file. cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ - # copy the aznfsclient binary. - cp -avf ${aznfsclient} ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient + if [ "$rpm_dir" != "azurelinux" ]; then + # copy the aznfsclient binary. + cp -avf ${aznfsclient} ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient + else + # Define the final target location for aznfsclient + aznfsclient_target="${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient" - # - # Package aznfsclient dependencies in opt_dir/libs. - # libs_dir must already be populated with the required dependencies from - # the debian packaging step. Simply copy all those to rpm_libs_dir. - # - rpm_libs_dir=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/libs - mkdir -p ${rpm_libs_dir} - cp -avfH ${libs_dir}/* ${rpm_libs_dir} + # Copy the built aznfsclient binary to the target + cp -avf "${SOURCE_DIR}/turbonfs/build/aznfsclient" "${aznfsclient_target}" + + # Optional: fail early if copy fails + if [ ! -f "${aznfsclient_target}" ]; then + echo "Error: aznfsclient failed to copy to ${aznfsclient_target}" + exit 1 + fi + fi + + if [ "$rpm_dir" != "azurelinux" ]; then + # + # Package aznfsclient dependencies in opt_dir/libs. + # libs_dir must already be populated with the required dependencies from + # the debian packaging step. Simply copy all those to rpm_libs_dir. + # + rpm_libs_dir=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/libs + mkdir -p ${rpm_libs_dir} + cp -avfH ${libs_dir}/* ${rpm_libs_dir} + fi # Create the archive for the package. tar -cvzf ${STG_DIR}/${rpm_pkg_dir}.tar.gz -C ${STG_DIR}/${rpm_dir}/tmp root @@ -84,20 +107,22 @@ generate_rpm_package() # Copy the SPEC file to change the placeholders depending upon the RPM distro. cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/RPM/aznfs.spec ${STG_DIR}/${rpm_dir}/tmp/ - # - # Insert the contents of ${rpm_libs_dir}. - # This is variable due to the shared library versions. - # sed doesn't (easily) support replace target to be multi-line, so we use - # awk for this one. - # - opt_libs=$(for lib in ${rpm_libs_dir}/*; do echo ${opt_dir}/libs/$(basename $lib); done) - awk -i inplace -v r="$opt_libs" '{gsub(/OPT_LIBS/,r)}1' ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + if [ "$rpm_dir" != "azurelinux" ]; then + # + # Insert the contents of ${rpm_libs_dir}. + # This is variable due to the shared library versions. + # sed doesn't (easily) support replace target to be multi-line, so we use + # awk for this one. + # + opt_libs=$(for lib in ${rpm_libs_dir}/*; do echo ${opt_dir}/libs/$(basename $lib); done) + awk -i inplace -v r="$opt_libs" '{gsub(/OPT_LIBS/,r)}1' ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + fi # Insert current release number and RPM_DIR value. sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec sed -i -e "s/RPM_DIR/${rpm_dir}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec sed -i -e "s/BUILD_ARCH/${arch}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec - + # Replace the placeholders for various package names in aznfs.spec file. if [ "$rpm_dir" == "suse" ]; then sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}_sles/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec @@ -108,6 +133,8 @@ generate_rpm_package() else if [ "$rpm_dir" == "stunnel" ]; then sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}_stunnel_custom/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + elif [ "$rpm_dir" == "azurelinux" ]; then + sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}-azurelinux/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec else sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec fi @@ -121,7 +148,7 @@ generate_rpm_package() fi # Create the rpm package. - rpmbuild --define "custom_stunnel $custom_stunnel_required" --define "_topdir ${STG_DIR}/${rpm_dir}${rpmbuild_dir}" -v -bb ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + rpmbuild --define "custom_stunnel $custom_stunnel_required" --define "azurelinux_build $azurelinux_build_required" --define "_topdir ${STG_DIR}/${rpm_dir}${rpmbuild_dir}" -v -bb ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec } generate_tarball_package() @@ -203,29 +230,31 @@ rpm_buildroot_dir="${rpmbuild_dir}/BUILDROOT" # Insert release number to aznfs_install.sh sed -i -e "s/RELEASE_NUMBER=x.y.z/RELEASE_NUMBER=${RELEASE_NUMBER}/g" ${SOURCE_DIR}/scripts/aznfs_install.sh -######################### -# Generate .deb package # -######################### +if [ "$BUILD_MACHINE" != "azurelinux" ]; then + ######################### + # Generate .deb package # + ######################### -# Create the directory to hold the package control and data files for deb package. -mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN + # Create the directory to hold the package control and data files for deb package. + mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN -# Copy the debian control file(s) and maintainer scripts. -cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ -chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* + # Copy the debian control file(s) and maintainer scripts. + cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ + chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* -# Insert current release number. -sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control -sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control + # Insert current release number. + sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control + sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control -# Copy other static package file(s). -mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin -cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ -cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ + # Copy other static package file(s). + mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin + cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ + cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ -# Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. -mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin -gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs + # Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. + mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin + gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs +fi # # We build the turbonfs project here, note that we can set all cmake options in the @@ -246,6 +275,13 @@ else INSECURE_AUTH_FOR_DEVTEST=OFF fi +# Run azurelinux packaging only on azurelinux runner +if [ "$BUILD_MACHINE" == "azurelinux" ]; then + DYNAMIC_LINKS=ON +else + DYNAMIC_LINKS=OFF +fi + # vcpkg required env variable VCPKG_FORCE_SYSTEM_BINARIES to be set for arm64. if [ "$(uname -m)" == "aarch64" ]; then export VCPKG_FORCE_SYSTEM_BINARIES=1 @@ -254,72 +290,77 @@ fi cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DENABLE_PARANOID=${PARANOID} \ -DENABLE_INSECURE_AUTH_FOR_DEVTEST=${INSECURE_AUTH_FOR_DEVTEST} \ + -DENABLE_DYNAMIC_LINKS=${DYNAMIC_LINKS} \ -DPACKAGE_VERSION="${RELEASE_NUMBER}" \ -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake .. make popd -mkdir -p ${STG_DIR}/deb/${pkg_dir}${opt_dir} -cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/src/mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/src/nfsv3mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/src/nfsv4mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/scripts/aznfs_install.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/deb/${pkg_dir}/${opt_dir}/ +if [ "$BUILD_MACHINE" != "azurelinux" ]; then + mkdir -p ${STG_DIR}/deb/${pkg_dir}${opt_dir} + cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/nfsv3mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/nfsv4mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/scripts/aznfs_install.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/deb/${pkg_dir}/${opt_dir}/ -mkdir -p ${STG_DIR}/deb/${pkg_dir}${system_dir} -cp -avf ${SOURCE_DIR}/src/aznfswatchdog.service ${STG_DIR}/deb/${pkg_dir}${system_dir} -cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4.service ${STG_DIR}/deb/${pkg_dir}${system_dir} + mkdir -p ${STG_DIR}/deb/${pkg_dir}${system_dir} + cp -avf ${SOURCE_DIR}/src/aznfswatchdog.service ${STG_DIR}/deb/${pkg_dir}${system_dir} + cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4.service ${STG_DIR}/deb/${pkg_dir}${system_dir} -########################################### -# Bundle aznfsclient and its dependencies # -########################################### + ########################################### + # Bundle aznfsclient and its dependencies # + ########################################### -# aznfsclient in the final target dir. -aznfsclient=${STG_DIR}/deb/${pkg_dir}/sbin/aznfsclient -cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} + # aznfsclient in the final target dir. + aznfsclient=${STG_DIR}/deb/${pkg_dir}/sbin/aznfsclient + cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} -# Package aznfsclient dependencies in opt_dir. -libs_dir=${STG_DIR}/deb/${pkg_dir}${opt_dir}/libs -mkdir -p ${libs_dir} + # Package aznfsclient dependencies in opt_dir. + libs_dir=${STG_DIR}/deb/${pkg_dir}${opt_dir}/libs + mkdir -p ${libs_dir} -# Copy the dependencies. -cp -avfH $(ldd ${aznfsclient} | grep "=>" | awk '{print $3}') ${libs_dir} + # Copy the dependencies. + cp -avfH $(ldd ${aznfsclient} | grep "=>" | awk '{print $3}') ${libs_dir} -# -# Patch all the libs to reference shared libs from ${libs_dir}. -# This is our very simple containerization. -# -for lib in ${libs_dir}/*.so*; do - echo "Setting rpath to ${opt_dir}/libs for $lib" - patchelf --set-rpath ${opt_dir}/libs "$lib" -done + # + # Patch all the libs to reference shared libs from ${libs_dir}. + # This is our very simple containerization. + # + for lib in ${libs_dir}/*.so*; do + echo "Setting rpath to ${opt_dir}/libs for $lib" + patchelf --set-rpath ${opt_dir}/libs "$lib" + done -# -# Final containerization effort - bundle and use the same interpreter as the -# build machine. -# -ld_linux_path=$(ldd ${aznfsclient} | grep "ld-linux" | awk '{print $1}') -ld_linux_name=$(basename "$ld_linux_path") -ld_linux="${libs_dir}/${ld_linux_name}" -cp -avfH "${ld_linux_path}" "${ld_linux}" + # + # Final containerization effort - bundle and use the same interpreter as the + # build machine. + # + ld_linux_path=$(ldd ${aznfsclient} | grep "ld-linux" | awk '{print $1}') + ld_linux_name=$(basename "$ld_linux_path") + ld_linux="${libs_dir}/${ld_linux_name}" + cp -avfH "${ld_linux_path}" "${ld_linux}" -patchelf --set-interpreter ${opt_dir}/libs/${ld_linux_name} ${aznfsclient} + patchelf --set-interpreter ${opt_dir}/libs/${ld_linux_name} ${aznfsclient} -# Create the deb package. -dpkg-deb -Zgzip --root-owner-group --build $STG_DIR/deb/$pkg_dir + # Create the deb package. + dpkg-deb -Zgzip --root-owner-group --build $STG_DIR/deb/$pkg_dir -######################### -# Generate .rpm package # -######################### + ######################### + # Generate .rpm package # + ######################### -generate_rpm_package rpm -generate_rpm_package suse -# Generate rpm package with custom stunnel installation for Mariner, RedHat7, and Centos7. -generate_rpm_package stunnel + generate_rpm_package rpm + generate_rpm_package suse + # Generate rpm package with custom stunnel installation for Mariner, RedHat7, and Centos7. + generate_rpm_package stunnel -############################# -# Generating Tarball for AKS# -############################# + ############################# + # Generating Tarball for AKS# + ############################# -generate_tarball_package $arch + generate_tarball_package $arch +else + generate_rpm_package azurelinux +fi \ No newline at end of file diff --git a/package_azurelinux.sh b/package_azurelinux.sh new file mode 100644 index 000000000..03f72f296 --- /dev/null +++ b/package_azurelinux.sh @@ -0,0 +1,200 @@ +# Exit on error. +set -e + +# Debian uses amd64/arm64 in place of x86_64/aarch64. +if [ "$(uname -m)" == "x86_64" ]; then + arch="x86_64" + debarch="amd64" +elif [ "$(uname -m)" == "aarch64" ]; then + arch="aarch64" + debarch="arm64" +else + echo "Unsupported architecture: $(uname -m)" + exit 1 +fi + + +generate_rpm_package() +{ + rpm_dir=$1 + custom_stunnel_required=0 + azurelinux_build_required=0 + + # Overwrite rpm_pkg_dir in case of SUSE. + if [ "$rpm_dir" == "suse" ]; then + rpm_pkg_dir="${pkg_name}_sles-${RELEASE_NUMBER}-1.$arch" + fi + + # Overwrite rpm_pkg_dir in case of azurelinux. + if [ "$rpm_dir" == "azurelinux" ]; then + rpm_pkg_dir="${pkg_name}-azurelinux-${RELEASE_NUMBER}-1.$arch" + azurelinux_build_required=1 + fi + + # Overwrite rpm_pkg_dir in case of Mariner, RedHat7, and Centos7. + if [ "$rpm_dir" == "stunnel" ]; then + rpm_pkg_dir="${pkg_name}_stunnel_custom-${RELEASE_NUMBER}-1.$arch" + custom_stunnel_required=1 + fi + + # Create the directory to hold the package spec and data files for RPM package. + mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir} + + # Copy static package file(s). + mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/usr/sbin + cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/usr/sbin/ + cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/usr/sbin/ + + # Compile mount.aznfs.c and put the executable into ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin. + mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin + gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/mount.aznfs + + mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir} + cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/mountscript.sh ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/nfsv3mountscript.sh ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/nfsv4mountscript.sh ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/scripts/aznfs_install.sh ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ + + mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${system_dir} + cp -avf ${SOURCE_DIR}/src/aznfswatchdog.service ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${system_dir} + cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4.service ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${system_dir} + + ########################################### + # Bundle aznfsclient and its dependencies # + ########################################### + + # copy the aznfsclient config file. + cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ + + # copy the aznfsclient binary. + # cp -avf ${aznfsclient} ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient + # Define the final target location for aznfsclient + aznfsclient_target="${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient" + + # Copy the built aznfsclient binary to the target + cp -avf "${SOURCE_DIR}/turbonfs/build/aznfsclient" "${aznfsclient_target}" + + # Optional: fail early if copy fails + if [ ! -f "${aznfsclient_target}" ]; then + echo "Error: aznfsclient failed to copy to ${aznfsclient_target}" + exit 1 + fi + + if [ "$rpm_dir" != "azurelinux" ]; then + # + # Package aznfsclient dependencies in opt_dir/libs. + # libs_dir must already be populated with the required dependencies from + # the debian packaging step. Simply copy all those to rpm_libs_dir. + # + rpm_libs_dir=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/libs + mkdir -p ${rpm_libs_dir} + cp -avfH ${libs_dir}/* ${rpm_libs_dir} + fi + + # Create the archive for the package. + tar -cvzf ${STG_DIR}/${rpm_pkg_dir}.tar.gz -C ${STG_DIR}/${rpm_dir}/tmp root + + # Copy the SPEC file to change the placeholders depending upon the RPM distro. + cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/RPM/aznfs.spec ${STG_DIR}/${rpm_dir}/tmp/ + + if [ "$rpm_dir" != "azurelinux" ]; then + # + # Insert the contents of ${rpm_libs_dir}. + # This is variable due to the shared library versions. + # sed doesn't (easily) support replace target to be multi-line, so we use + # awk for this one. + # + opt_libs=$(for lib in ${rpm_libs_dir}/*; do echo ${opt_dir}/libs/$(basename $lib); done) + awk -i inplace -v r="$opt_libs" '{gsub(/OPT_LIBS/,r)}1' ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + fi + + # Insert current release number and RPM_DIR value. + sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + sed -i -e "s/RPM_DIR/${rpm_dir}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + sed -i -e "s/BUILD_ARCH/${arch}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + + # Replace the placeholders for various package names in aznfs.spec file. + if [ "$rpm_dir" == "suse" ]; then + sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}_sles/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + sed -i -e "s/NETCAT_PACKAGE_NAME/netcat-openbsd/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + # For SLES, sysvinit-tools provides pidof. + sed -i -e "s/PROCPS_PACKAGE_NAME/sysvinit-tools/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + sed -i -e "s/DISTRO/suse/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + else + if [ "$rpm_dir" == "stunnel" ]; then + sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}_stunnel_custom/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + elif [ "$rpm_dir" == "azurelinux" ]; then + sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}-azurelinux/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + else + sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + fi + + sed -i -e "s/NETCAT_PACKAGE_NAME/nmap-ncat/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + # In new versions of Centos/RedHat/Rocky, procps-ng provides pidof. For older versions, it is provided by sysvinit-tools but since it is not + # present in new versions, only install procps-ng which exists in all versions. + sed -i -e "s/PROCPS_PACKAGE_NAME/procps-ng/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + sed -i -e "s/DISTRO/rpm/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + sed -i -e "s/INSTALL_CMD/yum/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + fi + + # Create the rpm package. + rpmbuild --define "custom_stunnel $custom_stunnel_required" --define "azurelinux_build $azurelinux_build_required" --define "_topdir ${STG_DIR}/${rpm_dir}${rpmbuild_dir}" -v -bb ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec +} + +#STG_DIR, RELEASE_NUMBER and SOURCE_DIR will be taken as env var. +pkg_name="aznfs" +pkg_dir="${pkg_name}-${RELEASE_NUMBER}-1_$debarch" +rpm_pkg_dir="${pkg_name}-${RELEASE_NUMBER}-1.$arch" +opt_dir="/opt/microsoft/${pkg_name}" +system_dir="/lib/systemd/system" +rpmbuild_dir="/root/rpmbuild" +rpm_buildroot_dir="${rpmbuild_dir}/BUILDROOT" + +# Insert release number to aznfs_install.sh +sed -i -e "s/RELEASE_NUMBER=x.y.z/RELEASE_NUMBER=${RELEASE_NUMBER}/g" ${SOURCE_DIR}/scripts/aznfs_install.sh + +# +# We build the turbonfs project here, note that we can set all cmake options in the +# future using env variables. +# +pushd ${SOURCE_DIR}/turbonfs +export VCPKG_ROOT=${SOURCE_DIR}/turbonfs/extern/vcpkg +# We need to update the submodules before calling cmake as toolchain build expects it. +git submodule update --recursive --init +mkdir -p build && cd build + +if [ "${BUILD_TYPE}" == "Debug" ]; then + PARANOID=ON + INSECURE_AUTH_FOR_DEVTEST=ON +else + PARANOID=OFF + INSECURE_AUTH_FOR_DEVTEST=OFF +fi + +# Run azurelinux packaging only on azurelinux runner +if [ "$BUILD_MACHINE" == "azurelinux" ]; then + DYNAMIC_LINKS=ON +else + DYNAMIC_LINKS=OFF +fi + +# vcpkg required env variable VCPKG_FORCE_SYSTEM_BINARIES to be set for arm64. +if [ "$(uname -m)" == "aarch64" ]; then + export VCPKG_FORCE_SYSTEM_BINARIES=1 +fi +# export VCPKG_FORCE_SYSTEM_BINARIES=1 + +cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + -DENABLE_PARANOID=${PARANOID} \ + -DENABLE_INSECURE_AUTH_FOR_DEVTEST=${INSECURE_AUTH_FOR_DEVTEST} \ + -DENABLE_DYNAMIC_LINKS=${DYNAMIC_LINKS} \ + -DPACKAGE_VERSION="${RELEASE_NUMBER}" \ + -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake .. +make +popd + +# Run azurelinux packaging only on azurelinux runner +if [ "$BUILD_MACHINE" == "azurelinux" ]; then + generate_rpm_package azurelinux +fi diff --git a/packaging/aznfs/RPM/aznfs.spec b/packaging/aznfs/RPM/aznfs.spec index 8b4b381ec..11f086b3f 100644 --- a/packaging/aznfs/RPM/aznfs.spec +++ b/packaging/aznfs/RPM/aznfs.spec @@ -7,6 +7,10 @@ URL: https://github.com/Azure/AZNFS-mount/blob/main/README.md %if 0%{?custom_stunnel} Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, net-tools, binutils, kernel-headers, openssl, openssl-devel, gcc, make, wget Recommends: build-essential + +%elif 0%{?azurelinux_build} +Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, stunnel, net-tools, gnutls, jemalloc, libasan + %else Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, stunnel, net-tools %endif @@ -41,7 +45,9 @@ tar -xzvf ${STG_DIR}/AZNFS_PACKAGE_NAME-${RELEASE_NUMBER}-1.BUILD_ARCH.tar.gz -C /opt/microsoft/aznfs/aznfs_install.sh /lib/systemd/system/aznfswatchdog.service /lib/systemd/system/aznfswatchdogv4.service +%if !0%{?azurelinux_build} OPT_LIBS +%endif /opt/microsoft/aznfs/sample-turbo-config.yaml /sbin/aznfsclient diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index 6e8a554e8..ab94a6828 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -25,6 +25,9 @@ option(ENABLE_CHATTY "Enable super verbose logs" OFF) option(ENABLE_TCMALLOC "Use tcmalloc for malloc/free/new/delete" OFF) option(ENABLE_JEMALLOC "Use jemalloc for malloc/free/new/delete" ON) option(ENABLE_INSECURE_AUTH_FOR_DEVTEST "Enable AZAUTH for non-TLS connections" OFF) +option(ENABLE_DYNAMIC_LINKS "Use dynamic linking instead of bundling libs" OFF) + +message(STATUS "ENABLE_INSECURE_AUTH_FOR_DEVTEST = ${ENABLE_INSECURE_AUTH_FOR_DEVTEST}") # # Builds that make it to customers need to be extra careful about any unnecessary # logging. Some warning logs we have in our code are to attract developer @@ -144,16 +147,23 @@ if(NOT azure-storage-blobs-cpp_FOUND) message(FATAL_ERROR "azure-storage-blobs-cpp not found! Did you run cmake with -DCMAKE_TOOLCHAIN_FILE?") endif() +# Choose package manager based on ENABLE_DYNAMIC_LINKS +if(ENABLE_DYNAMIC_LINKS) + set(PKG_MGR tdnf) +else() + set(PKG_MGR apt) +endif() + # # Ensure pkg_search_module(), needed for configuring some packages. # find_package(PkgConfig QUIET) if(NOT PKG_CONFIG_FOUND) message(STATUS "pkg-config not found, trying to install pkg-config!") - execute_process(COMMAND sudo apt install -y pkg-config + execute_process(COMMAND sudo ${PKG_MGR} install -y pkg-config RESULT_VARIABLE PKGCONFIG_INSTALL_RESULT) if(NOT PKGCONFIG_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install pkg-config failed with ${PKGCONFIG_INSTALL_RESULT}, try installing pkg-config manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install pkg-config failed with ${PKGCONFIG_INSTALL_RESULT}, try installing pkg-config manually and then run cmake again") else() # Call once more to ensure above install completed fine and also # add the pkg_search_module() command. @@ -167,10 +177,10 @@ endif() find_program(MESON_EXECUTABLE meson QUIET) if(NOT MESON_FOUND) message(STATUS "meson not found, trying to install meson!") - execute_process(COMMAND sudo apt install -y meson + execute_process(COMMAND sudo ${PKG_MGR} install -y meson RESULT_VARIABLE MESON_INSTALL_RESULT) if(NOT MESON_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install meson failed with ${MESON_INSTALL_RESULT}, try installing meson manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install meson failed with ${MESON_INSTALL_RESULT}, try installing meson manually and then run cmake again") else() find_program(MESON_EXECUTABLE meson REQUIRED) endif() @@ -182,19 +192,31 @@ endif() find_program(NINJA_EXECUTABLE ninja QUIET) if(NOT NINJA_FOUND) message(STATUS "ninja not found, trying to install ninja-build!") - execute_process(COMMAND sudo apt install -y ninja-build + execute_process(COMMAND sudo ${PKG_MGR} install -y ninja-build RESULT_VARIABLE NINJA_INSTALL_RESULT) if(NOT NINJA_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install ninja-build failed with ${NINJA_INSTALL_RESULT}, try installing ninja-build manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ninja-build failed with ${NINJA_INSTALL_RESULT}, try installing ninja-build manually and then run cmake again") else() find_program(NINJA_EXECUTABLE ninja REQUIRED) endif() endif() -# Meson will install the fuse library and header files in the following paths. -set(fuse3_LIBRARY "/usr/local/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/libfuse3.a") +# Decide fuse3 paths based on ENABLE_DYNAMIC_LINKS. +if(ENABLE_DYNAMIC_LINKS) + # Dynamic: Meson installs directly to /usr/local/lib. + set(fuse3_LIBRARY "/usr/local/lib/libfuse3.a") +else() + # Static: Use multiarch layout. + set(fuse3_LIBRARY "/usr/local/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/libfuse3.a") +endif() + +# Include path remains the same for both cases. set(fuse3_INCLUDE_DIR "/usr/local/include") +# Meson will install the fuse library and header files in the following paths. +# set(fuse3_LIBRARY "/usr/local/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/libfuse3.a") +# set(fuse3_INCLUDE_DIR "/usr/local/include") + add_custom_command( OUTPUT ${fuse3_LIBRARY} COMMAND ${MESON_EXECUTABLE} setup ${LIBFUSE_BUILD_DIR} ${LIBFUSE_SOURCE_DIR} --default-library=static --buildtype=${MESON_BUILD_TYPE} @@ -210,10 +232,15 @@ add_custom_target(libfuse ALL DEPENDS ${fuse3_LIBRARY}) find_package(GnuTLS "3.4.6" QUIET) if(NOT GNUTLS_FOUND) message(STATUS "GnuTLS not found, trying to install gnutls-dev!") - execute_process(COMMAND sudo apt install -y gnutls-dev + if(PKG_MGR STREQUAL "apt") + set(GNUTLS_PKG "gnutls-dev") + else() + set(GNUTLS_PKG "gnutls-devel") + endif() + execute_process(COMMAND sudo ${PKG_MGR} install -y ${GNUTLS_PKG} RESULT_VARIABLE GNUTLS_INSTALL_RESULT) if(NOT GNUTLS_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install gnutls-dev failed with ${GNUTLS_INSTALL_RESULT}, try installing gnutls-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${GNUTLS_PKG} failed with ${GNUTLS_INSTALL_RESULT}, try installing ${GNUTLS_PKG} manually and then run cmake again") else() find_package(GnuTLS "3.4.6" REQUIRED) endif() @@ -229,11 +256,16 @@ endif() find_package(tcmalloc QUIET) if(NOT tcmalloc_FOUND) - message(STATUS "tcmalloc not found, trying to install libgoogle-perftools-dev!") - execute_process(COMMAND sudo apt install -y libgoogle-perftools-dev + if(PKG_MGR STREQUAL "apt") + set(TCMALLOC_PKG "libgoogle-perftools-dev") + else() + set(TCMALLOC_PKG "gperftools-devel") + endif() + message(STATUS "tcmalloc not found, trying to install ${TCMALLOC_PKG}!") + execute_process(COMMAND sudo ${PKG_MGR} install -y ${TCMALLOC_PKG} RESULT_VARIABLE tcmalloc_INSTALL_RESULT) if(NOT tcmalloc_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install libgoogle-perftools-dev failed with ${tcmalloc_INSTALL_RESULT}, try installing libgoogle-perftools-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${TCMALLOC_PKG} failed with ${tcmalloc_INSTALL_RESULT}, try installing ${TCMALLOC_PKG} manually and then run cmake again") else() # Call once more to ensure above install completed fine and also # will set tcmalloc_INCLUDE_DIR and tcmalloc_LIBRARY variables. @@ -256,20 +288,29 @@ endif() pkg_search_module(JEMALLOC QUIET jemalloc) if(NOT JEMALLOC_FOUND) - message(STATUS "jemalloc not found, trying to install libjemalloc-dev!") - execute_process(COMMAND sudo apt install -y libjemalloc-dev + if(${PKG_MGR} STREQUAL "tdnf") + set(JEMALLOC_PKG jemalloc-devel) + else() + set(JEMALLOC_PKG libjemalloc-dev) + endif() + message(STATUS "jemalloc not found, trying to install ${JEMALLOC_PKG}!") + execute_process(COMMAND sudo ${PKG_MGR} install -y ${JEMALLOC_PKG} RESULT_VARIABLE JEMALLOC_INSTALL_RESULT) if(NOT JEMALLOC_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install libjemalloc-dev failed with ${JEMALLOC_INSTALL_RESULT}, try installing libjemalloc-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${JEMALLOC_PKG} failed with ${JEMALLOC_INSTALL_RESULT}, try installing ${JEMALLOC_PKG} manually and then run cmake again") else() # Call once more to ensure above install completed fine and also - # will set JEMALLOC_LIBRARY_DIRS variable. + # will set JEMALLOC_LIBRARIES variable. pkg_search_module(JEMALLOC REQUIRED jemalloc) endif() endif() -# We want to link against the static jemalloc lib. -message(STATUS "Using jemalloc static lib ${JEMALLOC_LIBRARY_DIRS}/libjemalloc.a") +if(ENABLE_DYNAMIC_LINKS) + message(STATUS "Using jemalloc lib ${JEMALLOC_LIBRARIES}") +else() + # We want to link against the static jemalloc lib. + message(STATUS "Using jemalloc static lib ${JEMALLOC_LIBRARY_DIRS}") +endif() endif() # @@ -277,11 +318,16 @@ endif() # pkg_search_module(UUID QUIET uuid) if(NOT UUID_FOUND) - message(STATUS "uuid not found, trying to install uuid-dev!") - execute_process(COMMAND sudo apt install -y uuid-dev + if(PKG_MGR STREQUAL "apt") + set(UUID_PKG "uuid-dev") + else() + set(UUID_PKG "uuid-devel") + endif() + message(STATUS "uuid not found, trying to install ${UUID_PKG}!") + execute_process(COMMAND sudo ${PKG_MGR} install -y ${UUID_PKG} RESULT_VARIABLE UUID_INSTALL_RESULT) if(NOT UUID_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install uuid-dev failed with ${UUID_INSTALL_RESULT}, try installing uuid-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${UUID_PKG} failed with ${UUID_INSTALL_RESULT}, try installing ${UUID_PKG} manually and then run cmake again") else() pkg_search_module(UUID REQUIRED uuid) endif() @@ -295,11 +341,16 @@ message(STATUS "Using uuid include dir ${UUID_INCLUDE_DIRS}") # find_package(ZLIB QUIET) if(NOT ZLIB_FOUND) - message(STATUS "zlib not found, trying to install zlib1g-dev!") - execute_process(COMMAND sudo apt install -y zlib1g-dev + if(PKG_MGR STREQUAL "apt") + set(ZLIB_PKG "zlib1g-dev") + else() + set(ZLIB_PKG "zlib-devel") + endif() + message(STATUS "zlib not found, trying to install ${ZLIB_PKG}!") + execute_process(COMMAND sudo ${PKG_MGR} install -y ${ZLIB_PKG} RESULT_VARIABLE ZLIB_INSTALL_RESULT) if(NOT ZLIB_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install zlib1g-dev failed with ${ZLIB_INSTALL_RESULT}, try installing zlib1g-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${ZLIB_PKG} failed with ${ZLIB_INSTALL_RESULT}, try installing ${ZLIB_PKG} manually and then run cmake again") else() # Call once more to ensure above install completed fine and also # will set ZLIB_INCLUDE_DIR and ZLIB_LIBRARY variables. @@ -377,66 +428,85 @@ target_compile_options(${CMAKE_PROJECT_NAME} PRIVATE -Werror ) -# -# Libraries for statically linking gnutls. -# libp11-kit and libunistring do not have a static version available, so we are -# forced to use the shared version and bundle it. -# -set(CMAKE_FIND_LIBRARY_SUFFIXES .a) +# Only use static gnutls linking when ENABLE_DYNAMIC_LINKS is NOT set. +if(NOT ENABLE_DYNAMIC_LINKS) + # + # Libraries for statically linking gnutls. + # libp11-kit and libunistring do not have a static version available, so we are + # forced to use the shared version and bundle it. + # + set(CMAKE_FIND_LIBRARY_SUFFIXES .a) + + find_library(GNUTLS_STATIC_LIB libgnutls.a REQUIRED) + # + # arm64 libgnutls.a has multiple definitions for some symbols exported by + # libcrypto.a. The libcrypto.a object has some more symbols which are needed, + # so we delete the offending object from libgnutls.a before linking. + # + if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") + execute_process( + COMMAND bash -c "cp -vf ${GNUTLS_STATIC_LIB} /tmp/gnutls.a; ar dv /tmp/gnutls.a aes-aarch64.o" + COMMAND_ERROR_IS_FATAL ANY) + set(GNUTLS_STATIC_LIB /tmp/gnutls.a) + endif() -find_library(GNUTLS_STATIC_LIB libgnutls.a REQUIRED) -# -# arm64 libgnutls.a has multiple definitions for some symbols exported by -# libcrypto.a. The libcrypto.a object has some more symbols which are needed, -# so we delete the offending object from libgnutls.a before linking. -# -if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") -execute_process( - COMMAND bash -c "cp -vf ${GNUTLS_STATIC_LIB} /tmp/gnutls.a; ar dv /tmp/gnutls.a aes-aarch64.o" - COMMAND_ERROR_IS_FATAL ANY) -set(GNUTLS_STATIC_LIB /tmp/gnutls.a) -endif() - -find_library(HOGWEED_STATIC_LIB libhogweed.a REQUIRED) -find_library(NETTLE_STATIC_LIB libnettle.a REQUIRED) -find_library(TASN1_STATIC_LIB libtasn1.a REQUIRED) -find_library(IDN2_STATIC_LIB libidn2.a REQUIRED) -find_library(GMP_STATIC_LIB libgmp.a REQUIRED) -set(CMAKE_FIND_LIBRARY_SUFFIXES .so) -find_library(P11_KIT_SHARED_LIB libp11-kit.so REQUIRED) -find_library(UNISTRING_SHARED_LIB NAMES - libunistring.so - libunistring.so.5 - libunistring.so.2 - REQUIRED) - -set(GNUTLS_ALL_LIBRARIES - ${GNUTLS_STATIC_LIB} - ${HOGWEED_STATIC_LIB} - ${NETTLE_STATIC_LIB} - ${TASN1_STATIC_LIB} - ${IDN2_STATIC_LIB} - ${GMP_STATIC_LIB} - ${P11_KIT_SHARED_LIB} - ${UNISTRING_SHARED_LIB}) -message(STATUS "Using gnutls libraries: ${GNUTLS_ALL_LIBRARIES}") + find_library(HOGWEED_STATIC_LIB libhogweed.a REQUIRED) + find_library(NETTLE_STATIC_LIB libnettle.a REQUIRED) + find_library(TASN1_STATIC_LIB libtasn1.a REQUIRED) + find_library(IDN2_STATIC_LIB libidn2.a REQUIRED) + find_library(GMP_STATIC_LIB libgmp.a REQUIRED) + set(CMAKE_FIND_LIBRARY_SUFFIXES .so) + find_library(P11_KIT_SHARED_LIB libp11-kit.so REQUIRED) + find_library(UNISTRING_SHARED_LIB NAMES + libunistring.so + libunistring.so.5 + libunistring.so.2 + REQUIRED) + + set(GNUTLS_ALL_LIBRARIES + ${GNUTLS_STATIC_LIB} + ${HOGWEED_STATIC_LIB} + ${NETTLE_STATIC_LIB} + ${TASN1_STATIC_LIB} + ${IDN2_STATIC_LIB} + ${GMP_STATIC_LIB} + ${P11_KIT_SHARED_LIB} + ${UNISTRING_SHARED_LIB}) + message(STATUS "Using gnutls libraries: ${GNUTLS_ALL_LIBRARIES}") +endif() # All libraries. -target_link_libraries(${CMAKE_PROJECT_NAME} - -static-libgcc -static-libstdc++ - -Wl,-rpath,/opt/microsoft/aznfs/libs - ${ZLIB_LIBRARIES} - dl - pthread - nfs - # GNUTLS libraries are needed by static libnfs. - ${GNUTLS_ALL_LIBRARIES} - yaml-cpp - spdlog - Azure::azure-identity - Azure::azure-storage-blobs - nlohmann_json::nlohmann_json - uuid) +if(ENABLE_DYNAMIC_LINKS) + # Dynamic linking: No static flags, use GnuTLS::GnuTLS target + target_link_libraries(${CMAKE_PROJECT_NAME} + ${ZLIB_LIBRARIES} + dl + pthread + nfs + yaml-cpp + spdlog + Azure::azure-identity + Azure::azure-storage-blobs + nlohmann_json::nlohmann_json + uuid + GnuTLS::GnuTLS) +else() + # Static linking: Use static-libgcc/libstdc++ and explicit gnutls libraries + target_link_libraries(${CMAKE_PROJECT_NAME} + -static-libgcc -static-libstdc++ + -Wl,-rpath,/opt/microsoft/aznfs/libs + ${ZLIB_LIBRARIES} + dl + pthread + nfs + ${GNUTLS_ALL_LIBRARIES} + yaml-cpp + spdlog + Azure::azure-identity + Azure::azure-storage-blobs + nlohmann_json::nlohmann_json + uuid) +endif() if(ENABLE_TCMALLOC) target_link_libraries(${CMAKE_PROJECT_NAME} @@ -444,8 +514,21 @@ target_link_libraries(${CMAKE_PROJECT_NAME} endif() if(ENABLE_JEMALLOC) -target_link_libraries(${CMAKE_PROJECT_NAME} - ${JEMALLOC_LIBRARY_DIRS}/libjemalloc.a) + if(ENABLE_DYNAMIC_LINKS) + target_link_libraries(${CMAKE_PROJECT_NAME} + ${JEMALLOC_LIBRARIES}) + else() + target_link_libraries(${CMAKE_PROJECT_NAME} + ${JEMALLOC_LIBRARY_DIRS}/libjemalloc.a) + endif() endif() +# --- Debug: Print all static libraries linked --- +get_target_property(LINK_LIBS ${CMAKE_PROJECT_NAME} LINK_LIBRARIES) +message(STATUS "==== Libraries linked to ${CMAKE_PROJECT_NAME} ====") +foreach(lib IN LISTS LINK_LIBS) + message(STATUS " ${lib}") +endforeach() +message(STATUS "===============================================") + install(TARGETS ${CMAKE_PROJECT_NAME})