fix AppImage setup for fso and qtfred

 - generate appimage for qtfred if it's available
 - separate fso and qtfred install roots
 - move fso appimage files out of global space
 - fix qtfred appimage launch script
 - prefer X11 Qt backend for qtfred

Author: notimaginative

CMakeLists.txt

@@ -170,7 +170,8 @@ SET(GENERATED_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/generated_source)
 include(generateHeaders)
 
 if(FSO_BUILD_APPIMAGE)
-	include(appimage)
+	SET(BINARY_DESTINATION "bin")
+	SET(LIBRAY_DESTINATION "lib")
 endif()
 
 include(clang-tidy)

ci/linux/generate_appimage.sh

@@ -3,12 +3,23 @@
 INSTALL_FOLDER=$1
 
 # Install Freespace2 targets
-cmake -DCMAKE_INSTALL_PREFIX=$INSTALL_FOLDER -DCOMPONENT=Unspecified -P cmake_install.cmake
-cmake -DCMAKE_INSTALL_PREFIX=$INSTALL_FOLDER -DCOMPONENT=Freespace2 -P cmake_install.cmake
+cmake -DCMAKE_INSTALL_PREFIX=$INSTALL_FOLDER/Freespace2 -DCOMPONENT=Unspecified -P cmake_install.cmake
+cmake -DCMAKE_INSTALL_PREFIX=$INSTALL_FOLDER/Freespace2 -DCOMPONENT=Freespace2 -P cmake_install.cmake
 
 # We need to be a bit creative for determining the AppImage name since we don't want to hard-code the name
-FILENAME="$(find $INSTALL_FOLDER/bin -name 'fs2_open_*' -type f -printf "%f\n").AppImage"
-appimagetool -n $INSTALL_FOLDER "$INSTALL_FOLDER/$FILENAME"
+FILENAME="$(find $INSTALL_FOLDER/Freespace2/bin -name 'fs2_open_*' -type f -printf "%f\n").AppImage"
+appimagetool -n "$INSTALL_FOLDER/Freespace2" "$INSTALL_FOLDER/$FILENAME"
 chmod +x "$INSTALL_FOLDER/$FILENAME"
 
+# Maybe install qtFRED targets
+if [ -f qtfred/cmake_install.cmake ]; then
+	cmake -DCMAKE_INSTALL_PREFIX=$INSTALL_FOLDER/qtFRED -DCOMPONENT=Unspecified -P cmake_install.cmake
+	cmake -DCMAKE_INSTALL_PREFIX=$INSTALL_FOLDER/qtFRED -DCOMPONENT=qtFRED -P cmake_install.cmake
+
+	# We need to be a bit creative for determining the AppImage name since we don't want to hard-code the name
+	FILENAME="$(find $INSTALL_FOLDER/qtFRED/bin -iname 'qtfred_*' -type f -printf "%f\n").AppImage"
+	appimagetool -n "$INSTALL_FOLDER/qtFRED" "$INSTALL_FOLDER/$FILENAME"
+	chmod +x "$INSTALL_FOLDER/$FILENAME"
+fi
+
 ls -al $INSTALL_FOLDER

cmake/appimage.cmake

@@ -60,6 +60,25 @@ if (FSO_INSTALL_DEBUG_FILES)
 	endif()
 endif()
 
+if(FSO_BUILD_APPIMAGE)
+	configure_file("${CMAKE_CURRENT_LIST_DIR}/cmake/AppRun.in" "${CMAKE_CURRENT_BINARY_DIR}/AppRun.gen" @ONLY)
+	file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/AppRun-$<CONFIG>"
+		INPUT "${CMAKE_CURRENT_BINARY_DIR}/AppRun.gen")
+
+	install(PROGRAMS "${CMAKE_CURRENT_BINARY_DIR}/AppRun-$<CONFIG>" DESTINATION "." RENAME "AppRun"
+			COMPONENT "Freespace2")
+
+	configure_file("${CMAKE_CURRENT_LIST_DIR}/cmake/AppImage.desktop.in" "${CMAKE_CURRENT_BINARY_DIR}/AppImage.desktop.gen" @ONLY)
+	file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/AppImage-$<CONFIG>.desktop"
+		INPUT "${CMAKE_CURRENT_BINARY_DIR}/AppImage.desktop.gen")
+
+	install(FILES "${CMAKE_CURRENT_BINARY_DIR}/AppImage-$<CONFIG>.desktop" DESTINATION "." RENAME "fso.desktop"
+			COMPONENT "Freespace2")
+
+	install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/resources/app_icon.png" DESTINATION "."
+			COMPONENT "Freespace2")
+endif()
+
 enable_clang_tidy(Freespace2)
 
 INCLUDE(util)

freespace2/CMakeLists.txt

@@ -1,4 +1,12 @@
 #!/bin/bash
 HERE=$(dirname $(readlink -f "${0}"))
-export LD_LIBRARY_PATH=${HERE}/usr/lib:$LD_LIBRARY_PATH
-exec "${HERE}/usr/bin/$<TARGET_FILE_NAME:qtfred>" "$@"
+export LD_LIBRARY_PATH=${HERE}/lib:$LD_LIBRARY_PATH
+
+# Qt5 doesn't work so well with Wayland for us so prefer the use of X11 unless
+# another backend is already specified
+# (NOTE: the -platform option will override this so we don't need to test for it)
+if [ -z "$QT_QPA_PLATFORM" ]; then
+	export QT_QPA_PLATFORM="xcb;wayland"
+fi
+
+exec "${HERE}/bin/$<TARGET_FILE_NAME:qtfred>" "$@"