From 96359e8aec3ea3f295bbe5636e16be977ff3fa7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20Toma=C5=BEi=C4=8D?= Date: Mon, 25 Oct 2021 11:30:17 +0200 Subject: [PATCH 1/7] Fix compile ambiguity --- crnlib/crn_vector.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crnlib/crn_vector.cpp b/crnlib/crn_vector.cpp index 7f429513..479831c9 100644 --- a/crnlib/crn_vector.cpp +++ b/crnlib/crn_vector.cpp @@ -20,8 +20,8 @@ bool elemental_vector::increase_capacity(uint min_new_capacity, bool grow_hint, return true; ptr_bits_t new_capacity = min_new_capacity; - if ((grow_hint) && (!math::is_power_of_2(new_capacity))) - new_capacity = math::next_pow2(new_capacity); + if ((grow_hint) && (!math::is_power_of_2((uint64)new_capacity))) + new_capacity = math::next_pow2((uint64)new_capacity); CRNLIB_ASSERT(new_capacity && (new_capacity > m_capacity)); From 2505d30a2afd7e9fdc51f8fa244ad16a03612454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20Toma=C5=BEi=C4=8D?= Date: Mon, 25 Oct 2021 11:31:04 +0200 Subject: [PATCH 2/7] Include outside of namespace --- crnlib/crn_console.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crnlib/crn_console.h b/crnlib/crn_console.h index b6e7ab18..3580c1a3 100644 --- a/crnlib/crn_console.h +++ b/crnlib/crn_console.h @@ -6,6 +6,9 @@ #ifdef WIN32 #include #include +#elif defined(__GNUC__) +#include +#include #endif namespace crnlib { class dynamic_string; @@ -99,8 +102,6 @@ inline int crn_getch() { return _getch(); } #elif defined(__GNUC__) -#include -#include inline int crn_getch() { struct termios oldt, newt; int ch; From 4e8ac12bbbcce8d2f832ff618b804bcc837ba794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20Toma=C5=BEi=C4=8D?= Date: Mon, 25 Oct 2021 11:33:40 +0200 Subject: [PATCH 3/7] Use macos malloc and file related functions --- crnlib/crn_jpge.cpp | 4 ++++ crnlib/crn_mem.cpp | 8 +++++++- crnlib/crn_platform.h | 2 +- inc/crn_decomp.h | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/crnlib/crn_jpge.cpp b/crnlib/crn_jpge.cpp index 2a9cb185..888b5c8d 100644 --- a/crnlib/crn_jpge.cpp +++ b/crnlib/crn_jpge.cpp @@ -12,7 +12,11 @@ #include #include +#if !defined(__APPLE__) #include +#else +#include +#endif #include "crn_core.h" diff --git a/crnlib/crn_mem.cpp b/crnlib/crn_mem.cpp index d32ea08b..75a9ffe4 100644 --- a/crnlib/crn_mem.cpp +++ b/crnlib/crn_mem.cpp @@ -3,14 +3,20 @@ #include "crn_core.h" #include "crn_console.h" #include "../inc/crnlib.h" +#if !defined(__APPLE__) #include +#else +#include +#endif #if CRNLIB_USE_WIN32_API #include "crn_winhdr.h" #endif #define CRNLIB_MEM_STATS 0 -#if !CRNLIB_USE_WIN32_API +#if defined(__APPLE__) +#define _msize malloc_size +#elif !CRNLIB_USE_WIN32_API #define _msize malloc_usable_size #endif diff --git a/crnlib/crn_platform.h b/crnlib/crn_platform.h index cb37c264..2a4c8f2a 100644 --- a/crnlib/crn_platform.h +++ b/crnlib/crn_platform.h @@ -18,7 +18,7 @@ const bool c_crnlib_little_endian_platform = false; const bool c_crnlib_big_endian_platform = !c_crnlib_little_endian_platform; -#ifdef __GNUC__ +#if defined(__GNUC__) && !defined(__APPLE__) #define crn_fopen(pDstFile, f, m) *(pDstFile) = fopen64(f, m) #define crn_fseek fseeko64 #define crn_ftell ftello64 diff --git a/inc/crn_decomp.h b/inc/crn_decomp.h index 6be169ad..0045863e 100644 --- a/inc/crn_decomp.h +++ b/inc/crn_decomp.h @@ -1934,7 +1934,7 @@ static void* crnd_default_realloc(void* p, size_t size, size_t* pActual_size, bo #ifdef WIN32 *pActual_size = p_new ? ::_msize(p_new) : 0; #elif defined(__APPLE__) - *pActual_size = p_new ? malloc_size(p_new) : 0; + *pActual_size = p_new ? ::malloc_size(p_new) : 0; #else *pActual_size = p_new ? malloc_usable_size(p_new) : 0; #endif From f1da22a5eb5c3d3e57baf21185f26f71c7e6004c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20Toma=C5=BEi=C4=8D?= Date: Mon, 25 Oct 2021 11:35:19 +0200 Subject: [PATCH 4/7] Make pthreads work on macos Instead of `pthread_spinlock_t` use `os_unfair_lock_t` because macos is missing the pthread spinlock. `sem_init` is deprecated on macos and always returns `-1`. Use `sem_open` together with `sem_unlink` to open an exclusive sempahore in global namespace. Use `sem_wait` because `sem_timedWait` is missing on macos. --- crnlib/crn_threading_pthreads.cpp | 73 +++++++++++++++++++++++++------ crnlib/crn_threading_pthreads.h | 13 +++++- 2 files changed, 72 insertions(+), 14 deletions(-) diff --git a/crnlib/crn_threading_pthreads.cpp b/crnlib/crn_threading_pthreads.cpp index 9b1b8128..144079a7 100644 --- a/crnlib/crn_threading_pthreads.cpp +++ b/crnlib/crn_threading_pthreads.cpp @@ -11,7 +11,7 @@ #include "crn_winhdr.h" #endif -#ifdef __GNUC__ +#if defined(__GNUC__) && !defined(__APPLE__) #include #endif @@ -19,6 +19,10 @@ #include #endif +#if defined(__APPLE__) +#include +#endif + namespace crnlib { uint g_number_of_processors = 1; @@ -27,6 +31,8 @@ void crn_threading_init() { SYSTEM_INFO g_system_info; GetSystemInfo(&g_system_info); g_number_of_processors = math::maximum(1U, g_system_info.dwNumberOfProcessors); +#elif defined(__APPLE__) + g_number_of_processors = math::maximum(1, sysconf(_SC_NPROCESSORS_ONLN)); #elif defined(__GNUC__) g_number_of_processors = math::maximum(1, get_nprocs()); #else @@ -36,7 +42,7 @@ void crn_threading_init() { crn_thread_id_t crn_get_current_thread_id() { // FIXME: Not portable - return static_cast(pthread_self()); + return (crn_thread_id_t)(pthread_self()); } void crn_sleep(unsigned int milliseconds) { @@ -95,15 +101,35 @@ void mutex::set_spin_count(unsigned int count) { } semaphore::semaphore(long initialCount, long maximumCount, const char* pName) { - maximumCount, pName; CRNLIB_ASSERT(maximumCount >= initialCount); - if (sem_init(&m_sem, 0, initialCount)) { +#if !defined(__APPLE__) + maximumCount, pName; + m_sem = new sem_t(); + if (sem_init(m_sem, 0, initialCount)) { CRNLIB_FAIL("semaphore: sem_init() failed"); } +#else + m_name = pName ? pName : "semaphore"; + for(int i = 0; i < 256; i++) { + m_sem = sem_open(m_name, O_CREAT | O_EXCL, 0644, initialCount); + if (m_sem != SEM_FAILED) + { + break; + } + sem_unlink(m_name); + } + if (m_sem == SEM_FAILED) + { + CRNLIB_FAIL("semaphore: sem_open() failed"); + } +#endif } semaphore::~semaphore() { - sem_destroy(&m_sem); + sem_destroy(m_sem); +#if defined(__APPLE__) + sem_unlink(m_name); +#endif } void semaphore::release(long releaseCount) { @@ -112,12 +138,12 @@ void semaphore::release(long releaseCount) { int status = 0; #ifdef WIN32 if (1 == releaseCount) - status = sem_post(&m_sem); + status = sem_post(m_sem); else - status = sem_post_multiple(&m_sem, releaseCount); + status = sem_post_multiple(m_sem, releaseCount); #else while (releaseCount > 0) { - status = sem_post(&m_sem); + status = sem_post(m_sem); if (status) break; releaseCount--; @@ -134,12 +160,12 @@ void semaphore::try_release(long releaseCount) { #ifdef WIN32 if (1 == releaseCount) - sem_post(&m_sem); + sem_post(m_sem); else - sem_post_multiple(&m_sem, releaseCount); + sem_post_multiple(m_sem, releaseCount); #else while (releaseCount > 0) { - sem_post(&m_sem); + sem_post(m_sem); releaseCount--; } #endif @@ -148,12 +174,16 @@ void semaphore::try_release(long releaseCount) { bool semaphore::wait(uint32 milliseconds) { int status; if (milliseconds == cUINT32_MAX) { - status = sem_wait(&m_sem); + status = sem_wait(m_sem); } else { +#if !defined(__APPLE__) struct timespec interval; interval.tv_sec = milliseconds / 1000; interval.tv_nsec = (milliseconds % 1000) * 1000000L; - status = sem_timedwait(&m_sem, &interval); + status = sem_timedwait(m_sem, &interval); +#else + status = sem_wait(m_sem); +#endif } if (status) { @@ -167,25 +197,42 @@ bool semaphore::wait(uint32 milliseconds) { } spinlock::spinlock() { +#if !defined(__APPLE__) if (pthread_spin_init(&m_spinlock, 0)) { CRNLIB_FAIL("spinlock: pthread_spin_init() failed"); } +#else + m_lock = new os_unfair_lock(); + *m_lock = OS_UNFAIR_LOCK_INIT; +#endif } spinlock::~spinlock() { +#if !defined(__APPLE__) pthread_spin_destroy(&m_spinlock); +#else + delete m_lock; +#endif } void spinlock::lock() { +#if !defined(__APPLE__) if (pthread_spin_lock(&m_spinlock)) { CRNLIB_FAIL("spinlock: pthread_spin_lock() failed"); } +#else + os_unfair_lock_lock(m_lock); +#endif } void spinlock::unlock() { +#if !defined(__APPLE__) if (pthread_spin_unlock(&m_spinlock)) { CRNLIB_FAIL("spinlock: pthread_spin_unlock() failed"); } +#else + os_unfair_lock_unlock(m_lock); +#endif } task_pool::task_pool() diff --git a/crnlib/crn_threading_pthreads.h b/crnlib/crn_threading_pthreads.h index 42ad6ed9..50f3fb02 100644 --- a/crnlib/crn_threading_pthreads.h +++ b/crnlib/crn_threading_pthreads.h @@ -14,6 +14,10 @@ #include #include +#if defined(__APPLE__) +#include +#endif + namespace crnlib { // g_number_of_processors defaults to 1. Will be higher on multicore machines. extern uint g_number_of_processors; @@ -71,7 +75,10 @@ class semaphore { bool wait(uint32 milliseconds = cUINT32_MAX); private: - sem_t m_sem; + sem_t* m_sem; +#if defined(__APPLE__) + const char* m_name; +#endif }; class spinlock { @@ -83,7 +90,11 @@ class spinlock { void unlock(); private: +#if !defined(__APPLE__) pthread_spinlock_t m_spinlock; +#else + os_unfair_lock_t m_lock; +#endif }; class scoped_spinlock { From 3f6d9bc5a55cb1ddc05fb09f2472bfa814e414fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20Toma=C5=BEi=C4=8D?= Date: Mon, 25 Oct 2021 11:52:16 +0200 Subject: [PATCH 5/7] Use max 16 threads The commit `4bb735f7966eedf414aa8e24533967cffd5bbcdb` only fixed the threading limit on the decoder part. Use the limit also in the encoder. --- crnlib/crn_threading_pthreads.cpp | 1 + crnlib/crn_threading_win32.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/crnlib/crn_threading_pthreads.cpp b/crnlib/crn_threading_pthreads.cpp index 144079a7..92e38e53 100644 --- a/crnlib/crn_threading_pthreads.cpp +++ b/crnlib/crn_threading_pthreads.cpp @@ -38,6 +38,7 @@ void crn_threading_init() { #else g_number_of_processors = 1; #endif + g_number_of_processors = math::minimum(g_number_of_processors, task_pool::cMaxThreads); } crn_thread_id_t crn_get_current_thread_id() { diff --git a/crnlib/crn_threading_win32.cpp b/crnlib/crn_threading_win32.cpp index d1730a37..50c114e9 100644 --- a/crnlib/crn_threading_win32.cpp +++ b/crnlib/crn_threading_win32.cpp @@ -13,6 +13,7 @@ void crn_threading_init() { GetSystemInfo(&g_system_info); g_number_of_processors = math::maximum(1U, g_system_info.dwNumberOfProcessors); + g_number_of_processors = math::minimum(g_number_of_processors, task_pool::cMaxThreads); } crn_thread_id_t crn_get_current_thread_id() { From 4b7442d5c3074a13acc9d2ee952063b70660e46a Mon Sep 17 00:00:00 2001 From: Thomas Debesse Date: Fri, 10 Feb 2023 12:22:28 +0100 Subject: [PATCH 6/7] cmake: enforce C++ 11 standard to avoid build errors on macOS Avoids errors on macOS like: In file included from crunch/crnlib/crn_threading.h:7: crunch/crnlib/crn_threading_pthreads.h:177:40: warning: defaulted function definitions are a C++11 extension [-Wc++11-extensions] virtual ~executable_task( void ) = default; ^ crunch/crnlib/crn_threading_pthreads.h:253:11: error: exception specification of overriding function is more lax than base version virtual ~object_task( void ) = default; ^ crunch/crnlib/crn_threading_pthreads.h:177:13: note: overridden virtual function is here virtual ~executable_task( void ) = default; ^ crunch/crnlib/crn_threading_pthreads.h:253:34: warning: defaulted function definitions are a C++11 extension [-Wc++11-extensions] virtual ~object_task( void ) = default; ^ --- CMakeLists.txt | 2 ++ crnlib/Makefile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 50062ea9..52591ee4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,7 @@ cmake_minimum_required(VERSION 3.1) +set(CMAKE_CXX_STANDARD 11) + set(CRUNCH_PROJECT_NAME crunch) set(CRUNCH_LIBRARY_NAME crn) set(CRUNCH_EXE_NAME crunch) diff --git a/crnlib/Makefile b/crnlib/Makefile index 5804f1e0..9ab0dd66 100644 --- a/crnlib/Makefile +++ b/crnlib/Makefile @@ -7,7 +7,7 @@ COMPILE_CPU_OPTIONS = -march=core2 COMPILE_DEBUG_OPTIONS = -g COMPILE_OPTIMIZATION_OPTIONS = -O3 -fomit-frame-pointer -ffast-math -fno-math-errno -fno-strict-aliasing COMPILE_WARN_OPTIONS = -Wall -Wno-unused-value -Wno-unused -COMPILE_OPTIONS = $(COMPILE_CPU_OPTIONS) $(COMPILE_DEBUG_OPTIONS) $(COMPILE_OPTIMIZATION_OPTIONS) $(COMPILE_WARN_OPTIONS) +COMPILE_OPTIONS = -std=c++11 $(COMPILE_CPU_OPTIONS) $(COMPILE_DEBUG_OPTIONS) $(COMPILE_OPTIMIZATION_OPTIONS) $(COMPILE_WARN_OPTIONS) LINKER_OPTIONS = -lpthread -g From ec8eb7a415c973f5c3900beb13767eba7c5558ce Mon Sep 17 00:00:00 2001 From: Thomas Debesse Date: Fri, 10 Feb 2023 15:19:17 +0100 Subject: [PATCH 7/7] crnlib: make CRNLIB_FORCE_INLINE functions extern because the compiler will override anyway Avoids errors on macOS like: duplicate symbol __ZN6crnlib5utils10write_le64EPvy in: crn_arealist.o crn_assert.o Setting static instead of extern would also workaround the error but it's wrong and would raise warnings on other systems like FreeBSD: In file included from crunch/crnlib/crn_core.h:174: crunch/crnlib/crn_utils.h:235:8: warning: 'gnu_inline' attribute without 'extern' in C++ treated as externally available, this changed in Clang 10 [-Wgnu-inline-cpp-without-extern] static CRNLIB_FORCE_INLINE void write_be64(void* p, uint64 x) { ^ crunch/crnlib/crn_core.h:85:70: note: expanded from macro 'CRNLIB_FORCE_INLINE' #define CRNLIB_FORCE_INLINE inline __attribute__((__always_inline__, __gnu_inline__)) ^ Setting extern fixes the duplicate symbols error on macOS without introducing warnings on FreeBSD. --- crnlib/crn_utils.h | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/crnlib/crn_utils.h b/crnlib/crn_utils.h index eccaaade..3df7fbaa 100644 --- a/crnlib/crn_utils.h +++ b/crnlib/crn_utils.h @@ -157,82 +157,82 @@ inline bool read_obj(T& obj, const void*& pBuf, uint& buf_size, bool buffer_is_l } #if defined(_MSC_VER) -static CRNLIB_FORCE_INLINE uint16 swap16(uint16 x) { +extern CRNLIB_FORCE_INLINE uint16 swap16(uint16 x) { return _byteswap_ushort(x); } -static CRNLIB_FORCE_INLINE uint32 swap32(uint32 x) { +extern CRNLIB_FORCE_INLINE uint32 swap32(uint32 x) { return _byteswap_ulong(x); } -static CRNLIB_FORCE_INLINE uint64 swap64(uint64 x) { +extern CRNLIB_FORCE_INLINE uint64 swap64(uint64 x) { return _byteswap_uint64(x); } #elif defined(__GNUC__) -static CRNLIB_FORCE_INLINE uint16 swap16(uint16 x) { +extern CRNLIB_FORCE_INLINE uint16 swap16(uint16 x) { return static_cast((x << 8U) | (x >> 8U)); } -static CRNLIB_FORCE_INLINE uint32 swap32(uint32 x) { +extern CRNLIB_FORCE_INLINE uint32 swap32(uint32 x) { return __builtin_bswap32(x); } -static CRNLIB_FORCE_INLINE uint64 swap64(uint64 x) { +extern CRNLIB_FORCE_INLINE uint64 swap64(uint64 x) { return __builtin_bswap64(x); } #else -static CRNLIB_FORCE_INLINE uint16 swap16(uint16 x) { +extern CRNLIB_FORCE_INLINE uint16 swap16(uint16 x) { return static_cast((x << 8U) | (x >> 8U)); } -static CRNLIB_FORCE_INLINE uint32 swap32(uint32 x) { +extern CRNLIB_FORCE_INLINE uint32 swap32(uint32 x) { return ((x << 24U) | ((x << 8U) & 0x00FF0000U) | ((x >> 8U) & 0x0000FF00U) | (x >> 24U)); } -static CRNLIB_FORCE_INLINE uint64 swap64(uint64 x) { +extern CRNLIB_FORCE_INLINE uint64 swap64(uint64 x) { return (static_cast(swap32(static_cast(x))) << 32ULL) | swap32(static_cast(x >> 32U)); } #endif // Assumes x has been read from memory as a little endian value, converts to native endianness for manipulation. -CRNLIB_FORCE_INLINE uint16 swap_le16_to_native(uint16 x) { +extern CRNLIB_FORCE_INLINE uint16 swap_le16_to_native(uint16 x) { return c_crnlib_little_endian_platform ? x : swap16(x); } -CRNLIB_FORCE_INLINE uint32 swap_le32_to_native(uint32 x) { +extern CRNLIB_FORCE_INLINE uint32 swap_le32_to_native(uint32 x) { return c_crnlib_little_endian_platform ? x : swap32(x); } -CRNLIB_FORCE_INLINE uint64 swap_le64_to_native(uint64 x) { +extern CRNLIB_FORCE_INLINE uint64 swap_le64_to_native(uint64 x) { return c_crnlib_little_endian_platform ? x : swap64(x); } // Assumes x has been read from memory as a big endian value, converts to native endianness for manipulation. -CRNLIB_FORCE_INLINE uint16 swap_be16_to_native(uint16 x) { +extern CRNLIB_FORCE_INLINE uint16 swap_be16_to_native(uint16 x) { return c_crnlib_big_endian_platform ? x : swap16(x); } -CRNLIB_FORCE_INLINE uint32 swap_be32_to_native(uint32 x) { +extern CRNLIB_FORCE_INLINE uint32 swap_be32_to_native(uint32 x) { return c_crnlib_big_endian_platform ? x : swap32(x); } -CRNLIB_FORCE_INLINE uint64 swap_be64_to_native(uint64 x) { +extern CRNLIB_FORCE_INLINE uint64 swap_be64_to_native(uint64 x) { return c_crnlib_big_endian_platform ? x : swap64(x); } -CRNLIB_FORCE_INLINE uint32 read_le32(const void* p) { +extern CRNLIB_FORCE_INLINE uint32 read_le32(const void* p) { return swap_le32_to_native(*static_cast(p)); } -CRNLIB_FORCE_INLINE void write_le32(void* p, uint32 x) { +extern CRNLIB_FORCE_INLINE void write_le32(void* p, uint32 x) { *static_cast(p) = swap_le32_to_native(x); } -CRNLIB_FORCE_INLINE uint64 read_le64(const void* p) { +extern CRNLIB_FORCE_INLINE uint64 read_le64(const void* p) { return swap_le64_to_native(*static_cast(p)); } -CRNLIB_FORCE_INLINE void write_le64(void* p, uint64 x) { +extern CRNLIB_FORCE_INLINE void write_le64(void* p, uint64 x) { *static_cast(p) = swap_le64_to_native(x); } -CRNLIB_FORCE_INLINE uint32 read_be32(const void* p) { +extern CRNLIB_FORCE_INLINE uint32 read_be32(const void* p) { return swap_be32_to_native(*static_cast(p)); } -CRNLIB_FORCE_INLINE void write_be32(void* p, uint32 x) { +extern CRNLIB_FORCE_INLINE void write_be32(void* p, uint32 x) { *static_cast(p) = swap_be32_to_native(x); } -CRNLIB_FORCE_INLINE uint64 read_be64(const void* p) { +extern CRNLIB_FORCE_INLINE uint64 read_be64(const void* p) { return swap_be64_to_native(*static_cast(p)); } -CRNLIB_FORCE_INLINE void write_be64(void* p, uint64 x) { +extern CRNLIB_FORCE_INLINE void write_be64(void* p, uint64 x) { *static_cast(p) = swap_be64_to_native(x); }