diff --git a/.github/aznfs-build.yaml b/.github/aznfs-build.yaml index 5b4c92c1b..2a8b2db39 100644 --- a/.github/aznfs-build.yaml +++ b/.github/aznfs-build.yaml @@ -55,6 +55,7 @@ stages: export STG_DIR=$(System.DefaultWorkingDirectory) export SOURCE_DIR=$(System.DefaultWorkingDirectory) export BUILD_TYPE=${{ parameters.buildType }} + export BUILD_MACHINE=ubuntu chmod +x $SOURCE_DIR/generate_package.sh $SOURCE_DIR/generate_package.sh displayName: "Run Package Script" @@ -79,6 +80,49 @@ stages: PathtoPublish: $(Build.ArtifactStagingDirectory) artifactName: 'aznfs-temp' + - job: package_aznfs_azurelinux_amd64 + displayName: Package and Release ${{ parameters.versionName }} for azurelinux + pool: + name: 'aznfsazurelinuxdemopool' # Use your self-hosted agent + + steps: + - checkout: self + path: azurelinux/AZNFS-mount + displayName: 'Checkout repository' + + - script: | + echo "Installing prerequisites on Azure Linux..." + sudo tdnf update -y + sudo tdnf install -y glibc-static rpm-build perl perl-IPC-Cmd \ + gcc gcc-c++ make git curl unzip zip tar cmake \ + zlib zlib-devel libuuid-devel gnutls gnutls-devel nfs-utils \ + libyaml-devel spdlog-devel nlohmann-json-devel cmake make \ + pkgconf pkgconf-m4 pkgconf-pkg-config ninja-build binutils \ + kernel-headers kernel-devel autoconf automake libtool jemalloc-devel fuse3-devel + displayName: "Install prerequisites on Azure Linux" + + - script: | + export RELEASE_NUMBER=${{ parameters.versionName }} + export STG_DIR=$(System.DefaultWorkingDirectory) + export SOURCE_DIR=$(System.DefaultWorkingDirectory) + export BUILD_TYPE=${{ parameters.buildType }} + export BUILD_MACHINE=azurelinux + chmod +x $SOURCE_DIR/generate_package.sh + $SOURCE_DIR/generate_package.sh + displayName: "Run Package Script" + + - script: | + mkdir -p $(Build.ArtifactStagingDirectory)/artifacts/rpmazurelinux + cp -f $(System.DefaultWorkingDirectory)/azurelinux/root/rpmbuild/RPMS/x86_64/aznfs-${{ parameters.versionName }}-1.x86_64.rpm $(Build.ArtifactStagingDirectory)/artifacts/rpmazurelinux + echo "Listing Built Files..." + ls -R $(Build.ArtifactStagingDirectory) + displayName: "Copy RPM to Artifacts" + + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory) + artifactName: 'aznfs-temp' + - job: package_aznfs_arm64 displayName: Package and Release ${{ parameters.versionName }} for arm64 pool: @@ -103,6 +147,7 @@ stages: export STG_DIR=$(System.DefaultWorkingDirectory) export SOURCE_DIR=$(System.DefaultWorkingDirectory) export BUILD_TYPE=${{ parameters.buildType }} + export BUILD_MACHINE=ubuntu chmod +x $SOURCE_DIR/generate_package.sh $SOURCE_DIR/generate_package.sh displayName: "Run Package Script" @@ -126,6 +171,7 @@ stages: PathtoPublish: $(Build.ArtifactStagingDirectory) artifactName: 'aznfs-temp' +# TODO: add azurelinux later to release and publish stages post approvals - stage: ReleaseArtifacts displayName: "Sign and Release Artifacts" jobs: diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index ba2da2b12..47fcd2703 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -27,7 +27,7 @@ permissions: contents: write jobs: - package_mount_helper_amd64: + package_mount_helper_ubuntu_amd64: name: Package and Release ${{ github.event.inputs.versionName }} for amd64 (Test Release) runs-on: ubuntu-22.04 steps: @@ -54,7 +54,8 @@ jobs: export RELEASE_NUMBER=${{ github.event.inputs.versionName }} export STG_DIR=$GITHUB_WORKSPACE/amd64 export SOURCE_DIR=$GITHUB_WORKSPACE/amd64 - export BUILD_TYPE=${{ github.events.inputs.buildType }} + export BUILD_TYPE=${{ github.event.inputs.buildType }} + export BUILD_MACHINE=ubuntu chmod +x $SOURCE_DIR/package.sh $SOURCE_DIR/package.sh - name: Create Test Release @@ -74,7 +75,48 @@ jobs: body: | New Test Release ${{ github.event.inputs.versionName }} - package_mount_helper_arm64: + # NOTE: COMMENT THIS JOB IF AZURE LINUX RUNNER IS NOT CONNECTED + package_mount_helper_azurelinux_amd64: + name: Package and Release ${{ github.event.inputs.versionName }} for azurelinux + runs-on: self-hosted + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + path: azurelinux + + - name: Install prerequisites on Azure Linux + run: | + sudo tdnf update -y + sudo tdnf install -y glibc-static rpm-build perl perl-IPC-Cmd \ + gcc gcc-c++ make git curl unzip zip tar cmake \ + zlib zlib-devel libuuid-devel gnutls gnutls-devel nfs-utils \ + libyaml-devel spdlog-devel nlohmann-json-devel cmake make \ + pkgconf pkgconf-m4 pkgconf-pkg-config ninja-build binutils \ + kernel-headers kernel-devel autoconf automake libtool jemalloc-devel fuse3-devel + + - name: Run Package.sh for azurelinux + run: | + export RELEASE_NUMBER=${{ github.event.inputs.versionName }} + export STG_DIR=$GITHUB_WORKSPACE + export SOURCE_DIR=$GITHUB_WORKSPACE/azurelinux + export BUILD_TYPE=${{ github.event.inputs.buildType }} + export BUILD_MACHINE=azurelinux + chmod +x $SOURCE_DIR/package.sh + $SOURCE_DIR/package.sh + + - name: Create Release (azurelinux) + uses: softprops/action-gh-release@v1 + with: + name: Release ${{ github.event.inputs.versionName }} + tag_name: ${{ github.event.inputs.versionName }} + target_commitish: ${{ github.sha }} + files: | + ${{ github.workspace }}/azurelinux/root/rpmbuild/RPMS/x86_64/aznfs-azurelinux-${{ github.event.inputs.versionName }}-1.x86_64.rpm + body: | + New Release ${{ github.event.inputs.versionName }} + + package_mount_helper_ubuntu_arm64: name: Package and Release ${{ github.event.inputs.versionName }} for arm64 (Test Release) runs-on: ubuntu-22.04-arm steps: @@ -101,7 +143,7 @@ jobs: export RELEASE_NUMBER=${{ github.event.inputs.versionName }} export STG_DIR=$GITHUB_WORKSPACE/arm64 export SOURCE_DIR=$GITHUB_WORKSPACE/arm64 - export BUILD_TYPE=${{ github.events.inputs.buildType }} + export BUILD_TYPE=${{ github.event.inputs.buildType }} chmod +x $SOURCE_DIR/package.sh $SOURCE_DIR/package.sh - name: Create Test Release diff --git a/generate_package.sh b/generate_package.sh index 54d02649f..a1ae9f036 100644 --- a/generate_package.sh +++ b/generate_package.sh @@ -24,11 +24,11 @@ generate_rpm_package() { rpm_dir=$1 custom_stunnel_required=0 + azurelinux_build_required=0 - # Overwrite rpm_pkg_dir in case of RedHat7 and Centos7. - if [ "$rpm_dir" == "stunnel" ]; then - custom_stunnel_required=1 - fi + # Set flags based on rpm_dir type + [ "$rpm_dir" == "stunnel" ] && custom_stunnel_required=1 + [ "$rpm_dir" == "azurelinux" ] && azurelinux_build_required=1 # Create the directory to hold the package spec and data files for RPM package. mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir} @@ -40,7 +40,11 @@ generate_rpm_package() # Compile mount.aznfs.c and put the executable into ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin. mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin - gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/mount.aznfs + if [ "$rpm_dir" == "azurelinux" ]; then + gcc ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/mount.aznfs + else + gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/mount.aznfs + fi mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir} cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ @@ -60,17 +64,20 @@ generate_rpm_package() # copy the aznfsclient config file. cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ - # copy the aznfsclient binary. - cp -avf ${aznfsclient} ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient - - # - # Package aznfsclient dependencies in opt_dir/libs. - # libs_dir must already be populated with the required dependencies from - # the debian packaging step. Simply copy all those to rpm_libs_dir. - # - rpm_libs_dir=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/libs - mkdir -p ${rpm_libs_dir} - cp -avfH ${libs_dir}/* ${rpm_libs_dir} + # aznfsclient in the final target dir. + aznfsclient=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient + cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} + + if [ "$rpm_dir" != "azurelinux" ]; then + # + # Package aznfsclient dependencies in opt_dir/libs. + # libs_dir must already be populated with the required dependencies from + # the debian packaging step. Simply copy all those to rpm_libs_dir. + # + rpm_libs_dir=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/libs + mkdir -p ${rpm_libs_dir} + cp -avfH ${libs_dir}/* ${rpm_libs_dir} + fi # Create the archive for the package. tar -cvzf ${STG_DIR}/${rpm_pkg_dir}.tar.gz -C ${STG_DIR}/${rpm_dir}/tmp root @@ -78,14 +85,16 @@ generate_rpm_package() # Copy the SPEC file to change the placeholders depending upon the RPM distro. cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/RPM/aznfs.spec ${STG_DIR}/${rpm_dir}/tmp/ - # - # Insert the contents of ${rpm_libs_dir}. - # This is variable due to the shared library versions. - # sed doesn't (easily) support replace target to be multi-line, so we use - # awk for this one. - # - opt_libs=$(for lib in ${rpm_libs_dir}/*; do echo ${opt_dir}/libs/$(basename $lib); done) - awk -i inplace -v r="$opt_libs" '{gsub(/OPT_LIBS/,r)}1' ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + if [ "$rpm_dir" != "azurelinux" ]; then + # + # Insert the contents of ${rpm_libs_dir}. + # This is variable due to the shared library versions. + # sed doesn't (easily) support replace target to be multi-line, so we use + # awk for this one. + # + opt_libs=$(for lib in ${rpm_libs_dir}/*; do echo ${opt_dir}/libs/$(basename $lib); done) + awk -i inplace -v r="$opt_libs" '{gsub(/OPT_LIBS/,r)}1' ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + fi # Insert current release number and RPM_DIR value. sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec @@ -100,6 +109,7 @@ generate_rpm_package() sed -i -e "s/PROCPS_PACKAGE_NAME/sysvinit-tools/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec sed -i -e "s/DISTRO/suse/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec else + sed -i -e "s/NETCAT_PACKAGE_NAME/nmap-ncat/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec # In new versions of Centos/RedHat/Rocky, procps-ng provides pidof. For older versions, it is provided by sysvinit-tools but since it is not # present in new versions, only install procps-ng which exists in all versions. @@ -109,7 +119,7 @@ generate_rpm_package() fi # Create the rpm package. - rpmbuild --define "custom_stunnel $custom_stunnel_required" --define "_topdir ${STG_DIR}/${rpm_dir}${rpmbuild_dir}" -v -bb ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + rpmbuild --define "custom_stunnel $custom_stunnel_required" --define "azurelinux_build $azurelinux_build_required" --define "_topdir ${STG_DIR}/${rpm_dir}${rpmbuild_dir}" -v -bb ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec # Remove the temporary files. rm ${STG_DIR}/${rpm_pkg_dir}.tar.gz @@ -128,35 +138,6 @@ rpm_buildroot_dir="${rpmbuild_dir}/BUILDROOT" # Insert release number to aznfs_install.sh sed -i -e "s/RELEASE_NUMBER=x.y.z/RELEASE_NUMBER=${RELEASE_NUMBER}/g" ${SOURCE_DIR}/scripts/aznfs_install.sh -######################### -# Generate .deb package # -######################### - -# Create the directory to hold the package control and data files for deb package. -mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN - -# Copy the debian control file(s) and maintainer scripts. -cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ -chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* - -# Insert current release number. -sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control -sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control - -# Copy other static package file(s). -mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin -cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ -cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ - -# Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. -mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin -gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs - -# -# We build the turbonfs project here, note that we can set all cmake options in the -# future using env variables. -# - pushd ${SOURCE_DIR}/turbonfs export VCPKG_ROOT=${SOURCE_DIR}/turbonfs/extern/vcpkg # We need to update the submodules before calling cmake as toolchain build expects it. @@ -171,6 +152,13 @@ else INSECURE_AUTH_FOR_DEVTEST=OFF fi +# Run azurelinux packaging only on azurelinux runner +if [ "$BUILD_MACHINE" == "azurelinux" ]; then + DYNAMIC_LINKS=ON +else + DYNAMIC_LINKS=OFF +fi + # vcpkg required env variable VCPKG_FORCE_SYSTEM_BINARIES to be set for arm64. if [ "$(uname -m)" == "aarch64" ]; then export VCPKG_FORCE_SYSTEM_BINARIES=1 @@ -179,68 +167,97 @@ fi cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DENABLE_PARANOID=${PARANOID} \ -DENABLE_INSECURE_AUTH_FOR_DEVTEST=${INSECURE_AUTH_FOR_DEVTEST} \ + -DENABLE_DYNAMIC_LINKS=${DYNAMIC_LINKS} \ -DPACKAGE_VERSION="${RELEASE_NUMBER}" \ -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake .. make popd -mkdir -p ${STG_DIR}/deb/${pkg_dir}${opt_dir} -cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/src/mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/src/nfsv3mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/src/nfsv4mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/scripts/aznfs_install.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/deb/${pkg_dir}/${opt_dir}/ - -mkdir -p ${STG_DIR}/deb/${pkg_dir}${system_dir} -cp -avf ${SOURCE_DIR}/src/aznfswatchdog.service ${STG_DIR}/deb/${pkg_dir}${system_dir} -cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4.service ${STG_DIR}/deb/${pkg_dir}${system_dir} - -########################################### -# Bundle aznfsclient and its dependencies # -########################################### - -# aznfsclient in the final target dir. -aznfsclient=${STG_DIR}/deb/${pkg_dir}/sbin/aznfsclient -cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} - -# Package aznfsclient dependencies in opt_dir. -libs_dir=${STG_DIR}/deb/${pkg_dir}${opt_dir}/libs -mkdir -p ${libs_dir} - -# Copy the dependencies. -cp -avfH $(ldd ${aznfsclient} | grep "=>" | awk '{print $3}') ${libs_dir} - -# -# Patch all the libs to reference shared libs from ${libs_dir}. -# This is our very simple containerization. -# -for lib in ${libs_dir}/*.so*; do - echo "Setting rpath to ${opt_dir}/libs for $lib" - patchelf --set-rpath ${opt_dir}/libs "$lib" -done - -# -# Final containerization effort - bundle and use the same interpreter as the -# build machine. -# -ld_linux_path=$(ldd ${aznfsclient} | grep "ld-linux" | awk '{print $1}') -ld_linux_name=$(basename "$ld_linux_path") -ld_linux="${libs_dir}/${ld_linux_name}" -cp -avfH "${ld_linux_path}" "${ld_linux}" - -patchelf --set-interpreter ${opt_dir}/libs/${ld_linux_name} ${aznfsclient} - -# Create the deb package. -dpkg-deb -Zgzip --root-owner-group --build $STG_DIR/deb/$pkg_dir - -######################### -# Generate .rpm package # -######################### - -generate_rpm_package rpm -generate_rpm_package suse -# Generate rpm package with custom stunnel installation for RedHat7 and Centos7. -generate_rpm_package stunnel +if [ "$BUILD_MACHINE" != "azurelinux" ]; then + ######################### + # Generate .deb package # + ######################### + + # Create the directory to hold the package control and data files for deb package. + mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN + + # Copy the debian control file(s) and maintainer scripts. + cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ + chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* + + # Insert current release number. + sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control + sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control + + # Copy other static package file(s). + mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin + cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ + cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ + + # Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. + mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin + gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs + + mkdir -p ${STG_DIR}/deb/${pkg_dir}${opt_dir} + cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/nfsv3mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/nfsv4mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/scripts/aznfs_install.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/deb/${pkg_dir}/${opt_dir}/ + + mkdir -p ${STG_DIR}/deb/${pkg_dir}${system_dir} + cp -avf ${SOURCE_DIR}/src/aznfswatchdog.service ${STG_DIR}/deb/${pkg_dir}${system_dir} + cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4.service ${STG_DIR}/deb/${pkg_dir}${system_dir} + + ########################################### + # Bundle aznfsclient and its dependencies # + ########################################### + + # aznfsclient in the final target dir. + aznfsclient=${STG_DIR}/deb/${pkg_dir}/sbin/aznfsclient + cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} + + # Package aznfsclient dependencies in opt_dir. + libs_dir=${STG_DIR}/deb/${pkg_dir}${opt_dir}/libs + mkdir -p ${libs_dir} + + # Copy the dependencies. + cp -avfH $(ldd ${aznfsclient} | grep "=>" | awk '{print $3}') ${libs_dir} + + # + # Patch all the libs to reference shared libs from ${libs_dir}. + # This is our very simple containerization. + # + for lib in ${libs_dir}/*.so*; do + echo "Setting rpath to ${opt_dir}/libs for $lib" + patchelf --set-rpath ${opt_dir}/libs "$lib" + done + + # + # Final containerization effort - bundle and use the same interpreter as the + # build machine. + # + ld_linux_path=$(ldd ${aznfsclient} | grep "ld-linux" | awk '{print $1}') + ld_linux_name=$(basename "$ld_linux_path") + ld_linux="${libs_dir}/${ld_linux_name}" + cp -avfH "${ld_linux_path}" "${ld_linux}" + + patchelf --set-interpreter ${opt_dir}/libs/${ld_linux_name} ${aznfsclient} + + # Create the deb package. + dpkg-deb -Zgzip --root-owner-group --build $STG_DIR/deb/$pkg_dir + + ######################### + # Generate .rpm package # + ######################### + + generate_rpm_package rpm + generate_rpm_package suse + # Generate rpm package with custom stunnel installation for RedHat7 and Centos7. + generate_rpm_package stunnel +else + generate_rpm_package azurelinux +fi diff --git a/package.sh b/package.sh index 55eeb2524..8490327ad 100755 --- a/package.sh +++ b/package.sh @@ -24,12 +24,19 @@ generate_rpm_package() { rpm_dir=$1 custom_stunnel_required=0 + azurelinux_build_required=0 # Overwrite rpm_pkg_dir in case of SUSE. if [ "$rpm_dir" == "suse" ]; then rpm_pkg_dir="${pkg_name}_sles-${RELEASE_NUMBER}-1.$arch" fi + # Overwrite rpm_pkg_dir in case of azurelinux. + if [ "$rpm_dir" == "azurelinux" ]; then + rpm_pkg_dir="${pkg_name}-azurelinux-${RELEASE_NUMBER}-1.$arch" + azurelinux_build_required=1 + fi + # Overwrite rpm_pkg_dir in case of Mariner, RedHat7, and Centos7. if [ "$rpm_dir" == "stunnel" ]; then rpm_pkg_dir="${pkg_name}_stunnel_custom-${RELEASE_NUMBER}-1.$arch" @@ -46,7 +53,11 @@ generate_rpm_package() # Compile mount.aznfs.c and put the executable into ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin. mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin - gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/mount.aznfs + if [ "$rpm_dir" == "azurelinux" ]; then + gcc ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/mount.aznfs + else + gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/mount.aznfs + fi mkdir -p ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir} cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ @@ -66,17 +77,27 @@ generate_rpm_package() # copy the aznfsclient config file. cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/ - # copy the aznfsclient binary. - cp -avf ${aznfsclient} ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient + if [ "$rpm_dir" != "azurelinux" ]; then + # copy the aznfsclient binary. + cp -avf ${aznfsclient} ${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient + else + # Define the final target location for aznfsclient + aznfsclient_target="${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}/sbin/aznfsclient" - # - # Package aznfsclient dependencies in opt_dir/libs. - # libs_dir must already be populated with the required dependencies from - # the debian packaging step. Simply copy all those to rpm_libs_dir. - # - rpm_libs_dir=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/libs - mkdir -p ${rpm_libs_dir} - cp -avfH ${libs_dir}/* ${rpm_libs_dir} + # Copy the built aznfsclient binary to the target + cp -avf "${SOURCE_DIR}/turbonfs/build/aznfsclient" "${aznfsclient_target}" + fi + + if [ "$rpm_dir" != "azurelinux" ]; then + # + # Package aznfsclient dependencies in opt_dir/libs. + # libs_dir must already be populated with the required dependencies from + # the debian packaging step. Simply copy all those to rpm_libs_dir. + # + rpm_libs_dir=${STG_DIR}/${rpm_dir}/tmp${rpm_buildroot_dir}/${rpm_pkg_dir}${opt_dir}/libs + mkdir -p ${rpm_libs_dir} + cp -avfH ${libs_dir}/* ${rpm_libs_dir} + fi # Create the archive for the package. tar -cvzf ${STG_DIR}/${rpm_pkg_dir}.tar.gz -C ${STG_DIR}/${rpm_dir}/tmp root @@ -84,20 +105,22 @@ generate_rpm_package() # Copy the SPEC file to change the placeholders depending upon the RPM distro. cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/RPM/aznfs.spec ${STG_DIR}/${rpm_dir}/tmp/ - # - # Insert the contents of ${rpm_libs_dir}. - # This is variable due to the shared library versions. - # sed doesn't (easily) support replace target to be multi-line, so we use - # awk for this one. - # - opt_libs=$(for lib in ${rpm_libs_dir}/*; do echo ${opt_dir}/libs/$(basename $lib); done) - awk -i inplace -v r="$opt_libs" '{gsub(/OPT_LIBS/,r)}1' ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + if [ "$rpm_dir" != "azurelinux" ]; then + # + # Insert the contents of ${rpm_libs_dir}. + # This is variable due to the shared library versions. + # sed doesn't (easily) support replace target to be multi-line, so we use + # awk for this one. + # + opt_libs=$(for lib in ${rpm_libs_dir}/*; do echo ${opt_dir}/libs/$(basename $lib); done) + awk -i inplace -v r="$opt_libs" '{gsub(/OPT_LIBS/,r)}1' ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + fi # Insert current release number and RPM_DIR value. sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec sed -i -e "s/RPM_DIR/${rpm_dir}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec sed -i -e "s/BUILD_ARCH/${arch}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec - + # Replace the placeholders for various package names in aznfs.spec file. if [ "$rpm_dir" == "suse" ]; then sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}_sles/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec @@ -108,6 +131,8 @@ generate_rpm_package() else if [ "$rpm_dir" == "stunnel" ]; then sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}_stunnel_custom/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + elif [ "$rpm_dir" == "azurelinux" ]; then + sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}-azurelinux/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec else sed -i -e "s/AZNFS_PACKAGE_NAME/${pkg_name}/g" ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec fi @@ -121,7 +146,7 @@ generate_rpm_package() fi # Create the rpm package. - rpmbuild --define "custom_stunnel $custom_stunnel_required" --define "_topdir ${STG_DIR}/${rpm_dir}${rpmbuild_dir}" -v -bb ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec + rpmbuild --define "custom_stunnel $custom_stunnel_required" --define "azurelinux_build $azurelinux_build_required" --define "_topdir ${STG_DIR}/${rpm_dir}${rpmbuild_dir}" -v -bb ${STG_DIR}/${rpm_dir}/tmp/aznfs.spec } generate_tarball_package() @@ -203,30 +228,6 @@ rpm_buildroot_dir="${rpmbuild_dir}/BUILDROOT" # Insert release number to aznfs_install.sh sed -i -e "s/RELEASE_NUMBER=x.y.z/RELEASE_NUMBER=${RELEASE_NUMBER}/g" ${SOURCE_DIR}/scripts/aznfs_install.sh -######################### -# Generate .deb package # -######################### - -# Create the directory to hold the package control and data files for deb package. -mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN - -# Copy the debian control file(s) and maintainer scripts. -cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ -chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* - -# Insert current release number. -sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control -sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control - -# Copy other static package file(s). -mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin -cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ -cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ - -# Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. -mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin -gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs - # # We build the turbonfs project here, note that we can set all cmake options in the # future using env variables. @@ -246,6 +247,13 @@ else INSECURE_AUTH_FOR_DEVTEST=OFF fi +# Run azurelinux packaging only on azurelinux runner +if [ "$BUILD_MACHINE" == "azurelinux" ]; then + DYNAMIC_LINKS=ON +else + DYNAMIC_LINKS=OFF +fi + # vcpkg required env variable VCPKG_FORCE_SYSTEM_BINARIES to be set for arm64. if [ "$(uname -m)" == "aarch64" ]; then export VCPKG_FORCE_SYSTEM_BINARIES=1 @@ -254,72 +262,101 @@ fi cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DENABLE_PARANOID=${PARANOID} \ -DENABLE_INSECURE_AUTH_FOR_DEVTEST=${INSECURE_AUTH_FOR_DEVTEST} \ + -DENABLE_DYNAMIC_LINKS=${DYNAMIC_LINKS} \ -DPACKAGE_VERSION="${RELEASE_NUMBER}" \ -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake .. make popd -mkdir -p ${STG_DIR}/deb/${pkg_dir}${opt_dir} -cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/src/mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/src/nfsv3mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/src/nfsv4mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/scripts/aznfs_install.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ -cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/deb/${pkg_dir}/${opt_dir}/ +if [ "$BUILD_MACHINE" != "azurelinux" ]; then + ######################### + # Generate .deb package # + ######################### -mkdir -p ${STG_DIR}/deb/${pkg_dir}${system_dir} -cp -avf ${SOURCE_DIR}/src/aznfswatchdog.service ${STG_DIR}/deb/${pkg_dir}${system_dir} -cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4.service ${STG_DIR}/deb/${pkg_dir}${system_dir} + # Create the directory to hold the package control and data files for deb package. + mkdir -p ${STG_DIR}/deb/${pkg_dir}/DEBIAN -########################################### -# Bundle aznfsclient and its dependencies # -########################################### + # Copy the debian control file(s) and maintainer scripts. + cp -avf ${SOURCE_DIR}/packaging/${pkg_name}/DEBIAN/* ${STG_DIR}/deb/${pkg_dir}/DEBIAN/ + chmod +x ${STG_DIR}/deb/${pkg_dir}/DEBIAN/* -# aznfsclient in the final target dir. -aznfsclient=${STG_DIR}/deb/${pkg_dir}/sbin/aznfsclient -cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} + # Insert current release number. + sed -i -e "s/Version: x.y.z/Version: ${RELEASE_NUMBER}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control + sed -i -e "s/BUILD_ARCH/${debarch}/g" ${STG_DIR}/deb/${pkg_dir}/DEBIAN/control -# Package aznfsclient dependencies in opt_dir. -libs_dir=${STG_DIR}/deb/${pkg_dir}${opt_dir}/libs -mkdir -p ${libs_dir} + # Copy other static package file(s). + mkdir -p ${STG_DIR}/deb/${pkg_dir}/usr/sbin + cp -avf ${SOURCE_DIR}/src/aznfswatchdog ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ + cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4 ${STG_DIR}/deb/${pkg_dir}/usr/sbin/ -# Copy the dependencies. -cp -avfH $(ldd ${aznfsclient} | grep "=>" | awk '{print $3}') ${libs_dir} + # Compile mount.aznfs.c and put the executable into ${STG_DIR}/deb/${pkg_dir}/sbin. + mkdir -p ${STG_DIR}/deb/${pkg_dir}/sbin + gcc -static ${SOURCE_DIR}/src/mount.aznfs.c -o ${STG_DIR}/deb/${pkg_dir}/sbin/mount.aznfs -# -# Patch all the libs to reference shared libs from ${libs_dir}. -# This is our very simple containerization. -# -for lib in ${libs_dir}/*.so*; do - echo "Setting rpath to ${opt_dir}/libs for $lib" - patchelf --set-rpath ${opt_dir}/libs "$lib" -done + mkdir -p ${STG_DIR}/deb/${pkg_dir}${opt_dir} + cp -avf ${SOURCE_DIR}/lib/common.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/nfsv3mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/src/nfsv4mountscript.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/scripts/aznfs_install.sh ${STG_DIR}/deb/${pkg_dir}${opt_dir}/ + cp -avf ${SOURCE_DIR}/turbonfs/sample-turbo-config.yaml ${STG_DIR}/deb/${pkg_dir}/${opt_dir}/ -# -# Final containerization effort - bundle and use the same interpreter as the -# build machine. -# -ld_linux_path=$(ldd ${aznfsclient} | grep "ld-linux" | awk '{print $1}') -ld_linux_name=$(basename "$ld_linux_path") -ld_linux="${libs_dir}/${ld_linux_name}" -cp -avfH "${ld_linux_path}" "${ld_linux}" + mkdir -p ${STG_DIR}/deb/${pkg_dir}${system_dir} + cp -avf ${SOURCE_DIR}/src/aznfswatchdog.service ${STG_DIR}/deb/${pkg_dir}${system_dir} + cp -avf ${SOURCE_DIR}/src/aznfswatchdogv4.service ${STG_DIR}/deb/${pkg_dir}${system_dir} + + ########################################### + # Bundle aznfsclient and its dependencies # + ########################################### -patchelf --set-interpreter ${opt_dir}/libs/${ld_linux_name} ${aznfsclient} + # aznfsclient in the final target dir. + aznfsclient=${STG_DIR}/deb/${pkg_dir}/sbin/aznfsclient + cp -avf ${SOURCE_DIR}/turbonfs/build/aznfsclient ${aznfsclient} -# Create the deb package. -dpkg-deb -Zgzip --root-owner-group --build $STG_DIR/deb/$pkg_dir + # Package aznfsclient dependencies in opt_dir. + libs_dir=${STG_DIR}/deb/${pkg_dir}${opt_dir}/libs + mkdir -p ${libs_dir} -######################### -# Generate .rpm package # -######################### + # Copy the dependencies. + cp -avfH $(ldd ${aznfsclient} | grep "=>" | awk '{print $3}') ${libs_dir} -generate_rpm_package rpm -generate_rpm_package suse -# Generate rpm package with custom stunnel installation for Mariner, RedHat7, and Centos7. -generate_rpm_package stunnel + # + # Patch all the libs to reference shared libs from ${libs_dir}. + # This is our very simple containerization. + # + for lib in ${libs_dir}/*.so*; do + echo "Setting rpath to ${opt_dir}/libs for $lib" + patchelf --set-rpath ${opt_dir}/libs "$lib" + done + + # + # Final containerization effort - bundle and use the same interpreter as the + # build machine. + # + ld_linux_path=$(ldd ${aznfsclient} | grep "ld-linux" | awk '{print $1}') + ld_linux_name=$(basename "$ld_linux_path") + ld_linux="${libs_dir}/${ld_linux_name}" + cp -avfH "${ld_linux_path}" "${ld_linux}" -############################# -# Generating Tarball for AKS# -############################# + patchelf --set-interpreter ${opt_dir}/libs/${ld_linux_name} ${aznfsclient} -generate_tarball_package $arch + # Create the deb package. + dpkg-deb -Zgzip --root-owner-group --build $STG_DIR/deb/$pkg_dir + + ######################### + # Generate .rpm package # + ######################### + + generate_rpm_package rpm + generate_rpm_package suse + # Generate rpm package with custom stunnel installation for Mariner, RedHat7, and Centos7. + generate_rpm_package stunnel + + ############################# + # Generating Tarball for AKS# + ############################# + + generate_tarball_package $arch +else + generate_rpm_package azurelinux +fi \ No newline at end of file diff --git a/packaging/aznfs/RPM/aznfs.spec b/packaging/aznfs/RPM/aznfs.spec index 8b4b381ec..d847e5316 100644 --- a/packaging/aznfs/RPM/aznfs.spec +++ b/packaging/aznfs/RPM/aznfs.spec @@ -7,6 +7,10 @@ URL: https://github.com/Azure/AZNFS-mount/blob/main/README.md %if 0%{?custom_stunnel} Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, net-tools, binutils, kernel-headers, openssl, openssl-devel, gcc, make, wget Recommends: build-essential + +%elif 0%{?azurelinux_build} +Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, stunnel, net-tools, gnutls, jemalloc, libasan, fuse3 + %else Requires: bash, PROCPS_PACKAGE_NAME, conntrack-tools, iptables, bind-utils, iproute, util-linux, nfs-utils, NETCAT_PACKAGE_NAME, newt, stunnel, net-tools %endif @@ -41,7 +45,9 @@ tar -xzvf ${STG_DIR}/AZNFS_PACKAGE_NAME-${RELEASE_NUMBER}-1.BUILD_ARCH.tar.gz -C /opt/microsoft/aznfs/aznfs_install.sh /lib/systemd/system/aznfswatchdog.service /lib/systemd/system/aznfswatchdogv4.service +%if !0%{?azurelinux_build} OPT_LIBS +%endif /opt/microsoft/aznfs/sample-turbo-config.yaml /sbin/aznfsclient diff --git a/turbonfs/CMakeLists.txt b/turbonfs/CMakeLists.txt index 6e8a554e8..aa308f724 100644 --- a/turbonfs/CMakeLists.txt +++ b/turbonfs/CMakeLists.txt @@ -25,6 +25,8 @@ option(ENABLE_CHATTY "Enable super verbose logs" OFF) option(ENABLE_TCMALLOC "Use tcmalloc for malloc/free/new/delete" OFF) option(ENABLE_JEMALLOC "Use jemalloc for malloc/free/new/delete" ON) option(ENABLE_INSECURE_AUTH_FOR_DEVTEST "Enable AZAUTH for non-TLS connections" OFF) +option(ENABLE_DYNAMIC_LINKS "Use dynamic linking instead of bundling libs" OFF) + # # Builds that make it to customers need to be extra careful about any unnecessary # logging. Some warning logs we have in our code are to attract developer @@ -84,8 +86,11 @@ endif() set(INSTALL_BIN_DIR "${CMAKE_INSTALL_PREFIX}/bin" CACHE PATH "Installation directory for binaries") set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH}) -set(LIBFUSE_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/extern/libfuse") -set(LIBFUSE_BUILD_DIR "${CMAKE_BINARY_DIR}/extern/libfuse") +# Only set libfuse paths for static builds +if(NOT ENABLE_DYNAMIC_LINKS) + set(LIBFUSE_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/extern/libfuse") + set(LIBFUSE_BUILD_DIR "${CMAKE_BINARY_DIR}/extern/libfuse") +endif() # # We add the following submodule(s), update them as needed, before starting. @@ -122,8 +127,11 @@ if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/spdlog/CMakeLists.txt") message(FATAL_ERROR "The spdlog submodule was not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.") endif() -if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/libfuse/meson.build") - message(FATAL_ERROR "The libfuse submodule was not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.") +# Only check libfuse submodule for static builds +if(NOT ENABLE_DYNAMIC_LINKS) + if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/libfuse/meson.build") + message(FATAL_ERROR "The libfuse submodule was not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.") + endif() endif() if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/vcpkg/bootstrap-vcpkg.sh") @@ -144,16 +152,23 @@ if(NOT azure-storage-blobs-cpp_FOUND) message(FATAL_ERROR "azure-storage-blobs-cpp not found! Did you run cmake with -DCMAKE_TOOLCHAIN_FILE?") endif() +# Choose package manager based on ENABLE_DYNAMIC_LINKS +if(ENABLE_DYNAMIC_LINKS) + set(PKG_MGR tdnf) +else() + set(PKG_MGR apt) +endif() + # # Ensure pkg_search_module(), needed for configuring some packages. # find_package(PkgConfig QUIET) if(NOT PKG_CONFIG_FOUND) message(STATUS "pkg-config not found, trying to install pkg-config!") - execute_process(COMMAND sudo apt install -y pkg-config + execute_process(COMMAND sudo ${PKG_MGR} install -y pkg-config RESULT_VARIABLE PKGCONFIG_INSTALL_RESULT) if(NOT PKGCONFIG_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install pkg-config failed with ${PKGCONFIG_INSTALL_RESULT}, try installing pkg-config manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install pkg-config failed with ${PKGCONFIG_INSTALL_RESULT}, try installing pkg-config manually and then run cmake again") else() # Call once more to ensure above install completed fine and also # add the pkg_search_module() command. @@ -162,47 +177,90 @@ if(NOT PKG_CONFIG_FOUND) endif() # -# We need meson to compile libfuse. +# Handle FUSE3 based on build type # -find_program(MESON_EXECUTABLE meson QUIET) -if(NOT MESON_FOUND) - message(STATUS "meson not found, trying to install meson!") - execute_process(COMMAND sudo apt install -y meson - RESULT_VARIABLE MESON_INSTALL_RESULT) - if(NOT MESON_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install meson failed with ${MESON_INSTALL_RESULT}, try installing meson manually and then run cmake again") - else() - find_program(MESON_EXECUTABLE meson REQUIRED) +if(ENABLE_DYNAMIC_LINKS) + # + # For dynamic builds, use system-installed fuse3 shared library + # + message(STATUS "Using system-installed fuse3 (dynamic linking)") + + # First try to find fuse3 via pkg-config + find_package(PkgConfig REQUIRED) + pkg_search_module(FUSE3 QUIET fuse3) + + if(NOT FUSE3_FOUND) + message(STATUS "fuse3 not found via pkg-config, trying to install the devel package") + + if(PKG_MGR STREQUAL "apt") + set(FUSE3_PKG "libfuse3-dev") + else() + set(FUSE3_PKG "fuse3-devel") + endif() + + execute_process(COMMAND sudo ${PKG_MGR} install -y ${FUSE3_PKG} + RESULT_VARIABLE FUSE3_INSTALL_RESULT) + + if(NOT FUSE3_INSTALL_RESULT EQUAL "0") + message(FATAL_ERROR "Failed to install ${FUSE3_PKG}, please install manually") + endif() + + # Retry finding fuse3 after installation + pkg_search_module(FUSE3 REQUIRED fuse3) endif() -endif() -# -# We need ninja to install libfuse in build dir. -# -find_program(NINJA_EXECUTABLE ninja QUIET) -if(NOT NINJA_FOUND) - message(STATUS "ninja not found, trying to install ninja-build!") - execute_process(COMMAND sudo apt install -y ninja-build - RESULT_VARIABLE NINJA_INSTALL_RESULT) - if(NOT NINJA_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install ninja-build failed with ${NINJA_INSTALL_RESULT}, try installing ninja-build manually and then run cmake again") - else() - find_program(NINJA_EXECUTABLE ninja REQUIRED) + message(STATUS "Found fuse3 libraries: ${FUSE3_LIBRARIES}") + message(STATUS "Found fuse3 include dirs: ${FUSE3_INCLUDE_DIRS}") + message(STATUS "Found fuse3 library dirs: ${FUSE3_LIBRARY_DIRS}") +else() + # + # For static builds, build libfuse from source + # + message(STATUS "Building libfuse from source (static linking)") + + # + # We need meson to compile libfuse. + # + find_program(MESON_EXECUTABLE meson QUIET) + if(NOT MESON_FOUND) + message(STATUS "meson not found, trying to install meson!") + execute_process(COMMAND sudo ${PKG_MGR} install -y meson + RESULT_VARIABLE MESON_INSTALL_RESULT) + if(NOT MESON_INSTALL_RESULT EQUAL "0") + message(FATAL_ERROR "${PKG_MGR} install meson failed with ${MESON_INSTALL_RESULT}, try installing meson manually and then run cmake again") + else() + find_program(MESON_EXECUTABLE meson REQUIRED) + endif() + endif() + + # + # We need ninja to install libfuse in build dir. + # + find_program(NINJA_EXECUTABLE ninja QUIET) + if(NOT NINJA_FOUND) + message(STATUS "ninja not found, trying to install ninja-build!") + execute_process(COMMAND sudo ${PKG_MGR} install -y ninja-build + RESULT_VARIABLE NINJA_INSTALL_RESULT) + if(NOT NINJA_INSTALL_RESULT EQUAL "0") + message(FATAL_ERROR "${PKG_MGR} install ninja-build failed with ${NINJA_INSTALL_RESULT}, try installing ninja-build manually and then run cmake again") + else() + find_program(NINJA_EXECUTABLE ninja REQUIRED) + endif() endif() -endif() -# Meson will install the fuse library and header files in the following paths. -set(fuse3_LIBRARY "/usr/local/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/libfuse3.a") -set(fuse3_INCLUDE_DIR "/usr/local/include") + # Use multiarch layout for static builds + set(fuse3_LIBRARY "/usr/local/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/libfuse3.a") + set(fuse3_INCLUDE_DIR "/usr/local/include") -add_custom_command( - OUTPUT ${fuse3_LIBRARY} - COMMAND ${MESON_EXECUTABLE} setup ${LIBFUSE_BUILD_DIR} ${LIBFUSE_SOURCE_DIR} --default-library=static --buildtype=${MESON_BUILD_TYPE} - COMMAND ${MESON_EXECUTABLE} compile -C ${LIBFUSE_BUILD_DIR} - COMMAND sudo ${MESON_EXECUTABLE} install -C ${LIBFUSE_BUILD_DIR} -) + add_custom_command( + OUTPUT ${fuse3_LIBRARY} + COMMAND ${MESON_EXECUTABLE} setup ${LIBFUSE_BUILD_DIR} ${LIBFUSE_SOURCE_DIR} --default-library=static --buildtype=${MESON_BUILD_TYPE} + COMMAND ${MESON_EXECUTABLE} compile -C ${LIBFUSE_BUILD_DIR} + COMMAND sudo ${MESON_EXECUTABLE} install -C ${LIBFUSE_BUILD_DIR} + ) -add_custom_target(libfuse ALL DEPENDS ${fuse3_LIBRARY}) + add_custom_target(libfuse ALL DEPENDS ${fuse3_LIBRARY}) +endif() # # We need GnuTLS for secure libnfs transport. @@ -210,10 +268,15 @@ add_custom_target(libfuse ALL DEPENDS ${fuse3_LIBRARY}) find_package(GnuTLS "3.4.6" QUIET) if(NOT GNUTLS_FOUND) message(STATUS "GnuTLS not found, trying to install gnutls-dev!") - execute_process(COMMAND sudo apt install -y gnutls-dev + if(PKG_MGR STREQUAL "apt") + set(GNUTLS_PKG "gnutls-dev") + else() + set(GNUTLS_PKG "gnutls-devel") + endif() + execute_process(COMMAND sudo ${PKG_MGR} install -y ${GNUTLS_PKG} RESULT_VARIABLE GNUTLS_INSTALL_RESULT) if(NOT GNUTLS_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install gnutls-dev failed with ${GNUTLS_INSTALL_RESULT}, try installing gnutls-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${GNUTLS_PKG} failed with ${GNUTLS_INSTALL_RESULT}, try installing ${GNUTLS_PKG} manually and then run cmake again") else() find_package(GnuTLS "3.4.6" REQUIRED) endif() @@ -229,11 +292,16 @@ endif() find_package(tcmalloc QUIET) if(NOT tcmalloc_FOUND) - message(STATUS "tcmalloc not found, trying to install libgoogle-perftools-dev!") - execute_process(COMMAND sudo apt install -y libgoogle-perftools-dev + if(PKG_MGR STREQUAL "apt") + set(TCMALLOC_PKG "libgoogle-perftools-dev") + else() + set(TCMALLOC_PKG "gperftools-devel") + endif() + message(STATUS "tcmalloc not found, trying to install ${TCMALLOC_PKG}!") + execute_process(COMMAND sudo ${PKG_MGR} install -y ${TCMALLOC_PKG} RESULT_VARIABLE tcmalloc_INSTALL_RESULT) if(NOT tcmalloc_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install libgoogle-perftools-dev failed with ${tcmalloc_INSTALL_RESULT}, try installing libgoogle-perftools-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${TCMALLOC_PKG} failed with ${tcmalloc_INSTALL_RESULT}, try installing ${TCMALLOC_PKG} manually and then run cmake again") else() # Call once more to ensure above install completed fine and also # will set tcmalloc_INCLUDE_DIR and tcmalloc_LIBRARY variables. @@ -256,20 +324,29 @@ endif() pkg_search_module(JEMALLOC QUIET jemalloc) if(NOT JEMALLOC_FOUND) - message(STATUS "jemalloc not found, trying to install libjemalloc-dev!") - execute_process(COMMAND sudo apt install -y libjemalloc-dev + if(${PKG_MGR} STREQUAL "tdnf") + set(JEMALLOC_PKG jemalloc-devel) + else() + set(JEMALLOC_PKG libjemalloc-dev) + endif() + message(STATUS "jemalloc not found, trying to install ${JEMALLOC_PKG}!") + execute_process(COMMAND sudo ${PKG_MGR} install -y ${JEMALLOC_PKG} RESULT_VARIABLE JEMALLOC_INSTALL_RESULT) if(NOT JEMALLOC_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install libjemalloc-dev failed with ${JEMALLOC_INSTALL_RESULT}, try installing libjemalloc-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${JEMALLOC_PKG} failed with ${JEMALLOC_INSTALL_RESULT}, try installing ${JEMALLOC_PKG} manually and then run cmake again") else() # Call once more to ensure above install completed fine and also - # will set JEMALLOC_LIBRARY_DIRS variable. + # will set JEMALLOC_LIBRARIES variable. pkg_search_module(JEMALLOC REQUIRED jemalloc) endif() endif() -# We want to link against the static jemalloc lib. -message(STATUS "Using jemalloc static lib ${JEMALLOC_LIBRARY_DIRS}/libjemalloc.a") +if(ENABLE_DYNAMIC_LINKS) + message(STATUS "Using jemalloc lib ${JEMALLOC_LIBRARIES}") +else() + # We want to link against the static jemalloc lib. + message(STATUS "Using jemalloc static lib ${JEMALLOC_LIBRARY_DIRS}") +endif() endif() # @@ -277,11 +354,16 @@ endif() # pkg_search_module(UUID QUIET uuid) if(NOT UUID_FOUND) - message(STATUS "uuid not found, trying to install uuid-dev!") - execute_process(COMMAND sudo apt install -y uuid-dev + if(PKG_MGR STREQUAL "apt") + set(UUID_PKG "uuid-dev") + else() + set(UUID_PKG "uuid-devel") + endif() + message(STATUS "uuid not found, trying to install ${UUID_PKG}!") + execute_process(COMMAND sudo ${PKG_MGR} install -y ${UUID_PKG} RESULT_VARIABLE UUID_INSTALL_RESULT) if(NOT UUID_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install uuid-dev failed with ${UUID_INSTALL_RESULT}, try installing uuid-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${UUID_PKG} failed with ${UUID_INSTALL_RESULT}, try installing ${UUID_PKG} manually and then run cmake again") else() pkg_search_module(UUID REQUIRED uuid) endif() @@ -295,11 +377,16 @@ message(STATUS "Using uuid include dir ${UUID_INCLUDE_DIRS}") # find_package(ZLIB QUIET) if(NOT ZLIB_FOUND) - message(STATUS "zlib not found, trying to install zlib1g-dev!") - execute_process(COMMAND sudo apt install -y zlib1g-dev + if(PKG_MGR STREQUAL "apt") + set(ZLIB_PKG "zlib1g-dev") + else() + set(ZLIB_PKG "zlib-devel") + endif() + message(STATUS "zlib not found, trying to install ${ZLIB_PKG}!") + execute_process(COMMAND sudo ${PKG_MGR} install -y ${ZLIB_PKG} RESULT_VARIABLE ZLIB_INSTALL_RESULT) if(NOT ZLIB_INSTALL_RESULT EQUAL "0") - message(FATAL_ERROR "apt install zlib1g-dev failed with ${ZLIB_INSTALL_RESULT}, try installing zlib1g-dev manually and then run cmake again") + message(FATAL_ERROR "${PKG_MGR} install ${ZLIB_PKG} failed with ${ZLIB_INSTALL_RESULT}, try installing ${ZLIB_PKG} manually and then run cmake again") else() # Call once more to ensure above install completed fine and also # will set ZLIB_INCLUDE_DIR and ZLIB_LIBRARY variables. @@ -350,7 +437,10 @@ else() set(sources ${sources} src/main.cpp) add_executable(${CMAKE_PROJECT_NAME} ${sources}) -add_dependencies(${CMAKE_PROJECT_NAME} libfuse) + # Only add libfuse dependency for static builds + if(NOT ENABLE_DYNAMIC_LINKS) + add_dependencies(${CMAKE_PROJECT_NAME} libfuse) + endif() endif() # All include directories. @@ -365,10 +455,15 @@ target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE "${PROJECT_SOURCE_DIR}/extern/spdlog/include") if(NOT ENABLE_NO_FUSE) -target_include_directories(${CMAKE_PROJECT_NAME} - PRIVATE "${fuse3_INCLUDE_DIR}") -target_link_libraries(${CMAKE_PROJECT_NAME} - ${fuse3_LIBRARY}) + if(ENABLE_DYNAMIC_LINKS) + # Use system fuse3 include directories + target_include_directories(${CMAKE_PROJECT_NAME} + PRIVATE ${FUSE3_INCLUDE_DIRS}) + else() + # Use built-from-source fuse3 include directory + target_include_directories(${CMAKE_PROJECT_NAME} + PRIVATE "${fuse3_INCLUDE_DIR}") + endif() endif() target_compile_options(${CMAKE_PROJECT_NAME} @@ -377,66 +472,95 @@ target_compile_options(${CMAKE_PROJECT_NAME} PRIVATE -Werror ) -# -# Libraries for statically linking gnutls. -# libp11-kit and libunistring do not have a static version available, so we are -# forced to use the shared version and bundle it. -# -set(CMAKE_FIND_LIBRARY_SUFFIXES .a) +# Only use static gnutls linking when ENABLE_DYNAMIC_LINKS is NOT set. +if(NOT ENABLE_DYNAMIC_LINKS) + # + # Libraries for statically linking gnutls. + # libp11-kit and libunistring do not have a static version available, so we are + # forced to use the shared version and bundle it. + # + set(CMAKE_FIND_LIBRARY_SUFFIXES .a) + + find_library(GNUTLS_STATIC_LIB libgnutls.a REQUIRED) + # + # arm64 libgnutls.a has multiple definitions for some symbols exported by + # libcrypto.a. The libcrypto.a object has some more symbols which are needed, + # so we delete the offending object from libgnutls.a before linking. + # + if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") + execute_process( + COMMAND bash -c "cp -vf ${GNUTLS_STATIC_LIB} /tmp/gnutls.a; ar dv /tmp/gnutls.a aes-aarch64.o" + COMMAND_ERROR_IS_FATAL ANY) + set(GNUTLS_STATIC_LIB /tmp/gnutls.a) + endif() -find_library(GNUTLS_STATIC_LIB libgnutls.a REQUIRED) -# -# arm64 libgnutls.a has multiple definitions for some symbols exported by -# libcrypto.a. The libcrypto.a object has some more symbols which are needed, -# so we delete the offending object from libgnutls.a before linking. -# -if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") -execute_process( - COMMAND bash -c "cp -vf ${GNUTLS_STATIC_LIB} /tmp/gnutls.a; ar dv /tmp/gnutls.a aes-aarch64.o" - COMMAND_ERROR_IS_FATAL ANY) -set(GNUTLS_STATIC_LIB /tmp/gnutls.a) -endif() - -find_library(HOGWEED_STATIC_LIB libhogweed.a REQUIRED) -find_library(NETTLE_STATIC_LIB libnettle.a REQUIRED) -find_library(TASN1_STATIC_LIB libtasn1.a REQUIRED) -find_library(IDN2_STATIC_LIB libidn2.a REQUIRED) -find_library(GMP_STATIC_LIB libgmp.a REQUIRED) -set(CMAKE_FIND_LIBRARY_SUFFIXES .so) -find_library(P11_KIT_SHARED_LIB libp11-kit.so REQUIRED) -find_library(UNISTRING_SHARED_LIB NAMES - libunistring.so - libunistring.so.5 - libunistring.so.2 - REQUIRED) - -set(GNUTLS_ALL_LIBRARIES - ${GNUTLS_STATIC_LIB} - ${HOGWEED_STATIC_LIB} - ${NETTLE_STATIC_LIB} - ${TASN1_STATIC_LIB} - ${IDN2_STATIC_LIB} - ${GMP_STATIC_LIB} - ${P11_KIT_SHARED_LIB} - ${UNISTRING_SHARED_LIB}) -message(STATUS "Using gnutls libraries: ${GNUTLS_ALL_LIBRARIES}") + find_library(HOGWEED_STATIC_LIB libhogweed.a REQUIRED) + find_library(NETTLE_STATIC_LIB libnettle.a REQUIRED) + find_library(TASN1_STATIC_LIB libtasn1.a REQUIRED) + find_library(IDN2_STATIC_LIB libidn2.a REQUIRED) + find_library(GMP_STATIC_LIB libgmp.a REQUIRED) + set(CMAKE_FIND_LIBRARY_SUFFIXES .so) + find_library(P11_KIT_SHARED_LIB libp11-kit.so REQUIRED) + find_library(UNISTRING_SHARED_LIB NAMES + libunistring.so + libunistring.so.5 + libunistring.so.2 + REQUIRED) + + set(GNUTLS_ALL_LIBRARIES + ${GNUTLS_STATIC_LIB} + ${HOGWEED_STATIC_LIB} + ${NETTLE_STATIC_LIB} + ${TASN1_STATIC_LIB} + ${IDN2_STATIC_LIB} + ${GMP_STATIC_LIB} + ${P11_KIT_SHARED_LIB} + ${UNISTRING_SHARED_LIB}) + message(STATUS "Using gnutls libraries: ${GNUTLS_ALL_LIBRARIES}") +endif() # All libraries. -target_link_libraries(${CMAKE_PROJECT_NAME} - -static-libgcc -static-libstdc++ - -Wl,-rpath,/opt/microsoft/aznfs/libs - ${ZLIB_LIBRARIES} - dl - pthread - nfs - # GNUTLS libraries are needed by static libnfs. - ${GNUTLS_ALL_LIBRARIES} - yaml-cpp - spdlog - Azure::azure-identity - Azure::azure-storage-blobs - nlohmann_json::nlohmann_json - uuid) +if(ENABLE_DYNAMIC_LINKS) + # Dynamic linking: No static flags, use GnuTLS::GnuTLS target + target_link_libraries(${CMAKE_PROJECT_NAME} + ${ZLIB_LIBRARIES} + dl + pthread + nfs + yaml-cpp + spdlog + Azure::azure-identity + Azure::azure-storage-blobs + nlohmann_json::nlohmann_json + uuid + GnuTLS::GnuTLS) + + # Link against system fuse3 (only if not building nofuse) + if(NOT ENABLE_NO_FUSE) + target_link_libraries(${CMAKE_PROJECT_NAME} ${FUSE3_LIBRARIES}) + endif() +else() + # Static linking: Use static-libgcc/libstdc++ and explicit gnutls libraries + target_link_libraries(${CMAKE_PROJECT_NAME} + -static-libgcc -static-libstdc++ + -Wl,-rpath,/opt/microsoft/aznfs/libs + ${ZLIB_LIBRARIES} + dl + pthread + nfs + ${GNUTLS_ALL_LIBRARIES} + yaml-cpp + spdlog + Azure::azure-identity + Azure::azure-storage-blobs + nlohmann_json::nlohmann_json + uuid) + + # Link against static fuse3 (only if not building nofuse) + if(NOT ENABLE_NO_FUSE) + target_link_libraries(${CMAKE_PROJECT_NAME} ${fuse3_LIBRARY}) + endif() +endif() if(ENABLE_TCMALLOC) target_link_libraries(${CMAKE_PROJECT_NAME} @@ -444,8 +568,13 @@ target_link_libraries(${CMAKE_PROJECT_NAME} endif() if(ENABLE_JEMALLOC) -target_link_libraries(${CMAKE_PROJECT_NAME} - ${JEMALLOC_LIBRARY_DIRS}/libjemalloc.a) + if(ENABLE_DYNAMIC_LINKS) + target_link_libraries(${CMAKE_PROJECT_NAME} + ${JEMALLOC_LIBRARIES}) + else() + target_link_libraries(${CMAKE_PROJECT_NAME} + ${JEMALLOC_LIBRARY_DIRS}/libjemalloc.a) + endif() endif() install(TARGETS ${CMAKE_PROJECT_NAME})