diff --git a/src/sfizz/FilePool.cpp b/src/sfizz/FilePool.cpp
index b18d5254..cc0ed7b8 100644
--- a/src/sfizz/FilePool.cpp
+++ b/src/sfizz/FilePool.cpp
@@ -378,19 +378,17 @@ void sfz::FilePool::loadingThread() noexcept
 {
     FilePromisePtr promise;
     while (!quitThread) {
+        workerBarrier.wait();
 
         if (emptyQueue) {
-            while(promiseQueue.try_pop(promise)) {
+            while (promiseQueue.try_pop(promise)) {
                 // We're just dequeuing
             }
             emptyQueue = false;
+            semEmptyQueueFinished.post();
             continue;
         }
 
-        std::error_code ec;
-        workerBarrier.wait(ec);
-        ASSERT(!ec);
-
         if (!promiseQueue.try_pop(promise)) {
             continue;
         }
@@ -483,12 +481,8 @@ uint32_t sfz::FilePool::getPreloadSize() const noexcept
 void sfz::FilePool::emptyFileLoadingQueues() noexcept
 {
     emptyQueue = true;
-    std::error_code ec;
-    workerBarrier.post(ec);
-    ASSERT(!ec);
-
-    while (emptyQueue)
-        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    workerBarrier.post();
+    semEmptyQueueFinished.wait();
 }
 
 void sfz::FilePool::waitForBackgroundLoading() noexcept
diff --git a/src/sfizz/FilePool.h b/src/sfizz/FilePool.h
index 1325168c..80e7f8a2 100644
--- a/src/sfizz/FilePool.h
+++ b/src/sfizz/FilePool.h
@@ -273,6 +273,7 @@ private:
     // Signals
     volatile bool quitThread { false };
     volatile bool emptyQueue { false };
+    RTSemaphore semEmptyQueueFinished;
     std::atomic<int> threadsLoading { 0 };
     RTSemaphore workerBarrier;
     RTSemaphore semClearingRequest;