diff --git a/src/sfizz/Config.h b/src/sfizz/Config.h
index 52d26a31..5713b385 100644
--- a/src/sfizz/Config.h
+++ b/src/sfizz/Config.h
@@ -36,7 +36,9 @@ namespace config {
     constexpr int defaultSamplesPerBlock { 1024 };
     constexpr int preloadSize { 8192 };
     constexpr int numChannels { 2 };
+    constexpr int numBackgroundThreads { 4 };
     constexpr int numVoices { 64 };
+    constexpr int maxVoices { 256 };
     constexpr int sustainCC { 64 };
     constexpr int halfCCThreshold { 64 };
     constexpr int centPerSemitone { 100 };
diff --git a/src/sfizz/FilePool.cpp b/src/sfizz/FilePool.cpp
index f1034e72..a28a188f 100644
--- a/src/sfizz/FilePool.cpp
+++ b/src/sfizz/FilePool.cpp
@@ -164,7 +164,7 @@ sfz::FilePromisePtr sfz::FilePool::getFilePromise(const std::string& filename) n
         promise->preloadedData = preloaded->second.preloadedData;
         promise->sampleRate = preloaded->second.sampleRate;
         promise->oversamplingFactor = oversamplingFactor;
-        promiseQueue.enqueue(promise);
+        promiseQueue.try_enqueue(promise);
     }
     return promise;
 }
@@ -196,28 +196,34 @@ void sfz::FilePool::loadingThread() noexcept
             continue;
         }
 
-        if (!promiseQueue.try_dequeue(promise)) {
-            std::this_thread::sleep_for(0.1ms);
+        if (!promiseQueue.wait_dequeue_timed(promise, 50ms)) {
             continue;
         }
 
-        fs::path file { rootDirectory / std::string(promise->filename) };
-        SndfileHandle sndFile(reinterpret_cast<const char*>(file.c_str()));
-        if (sndFile.error() != 0)
-            continue;
+        // The voice abandoned the promise already we just don't care
+        if (promise.use_count() != 1) {
+            fs::path file { rootDirectory / std::string(promise->filename) };
+            SndfileHandle sndFile(reinterpret_cast<const char*>(file.c_str()));
+            if (sndFile.error() != 0)
+                continue;
 
-        DBG("Loading file for " << promise->filename << " in the background");
-        const uint32_t frames = sndFile.frames();
-        promise->fileData = readFromFile<float>(sndFile, frames, oversamplingFactor);
-        promise->dataReady = true;
-        temporaryFilePromises.push_back(promise);
+            DBG("Loading file for " << promise->filename << " in the background");
+            const uint32_t frames = sndFile.frames();
+            promise->fileData = readFromFile<float>(sndFile, frames, oversamplingFactor);
+            promise->dataReady = true;
+        }
+
+        while (!filledPromiseQueue.try_enqueue(promise)) {
+            DBG("Error enqueuing the file for " << promise->filename << " in the filledPromiseQueue");
+            std::this_thread::sleep_for(1ms);
+        }
         promise.reset();
     }
 }
 
 void sfz::FilePool::clear()
 {
-    emptyFileLoadingQueue();
+    emptyFileLoadingQueues();
     preloadedFiles.clear();
     temporaryFilePromises.clear();
     promisesToClean.clear();
@@ -225,19 +231,23 @@ void sfz::FilePool::clear()
 
 void sfz::FilePool::cleanupPromises() noexcept
 {
-    if (temporaryFilePromises.empty())
-        return;
+    FilePromisePtr promise;
+    // Remove stuff from the filled queue and put them in a linear storage
+    while (filledPromiseQueue.try_dequeue(promise)) {
+        temporaryFilePromises.push_back(promise);
+        promise.reset();
+    }
 
-    auto promise = temporaryFilePromises.begin();
+    auto promiseIterator = temporaryFilePromises.begin();
     auto sentinel = temporaryFilePromises.end() - 1;
-    while (promise != temporaryFilePromises.end()) {
-        if (promise->use_count() == 1) {
-            promisesToClean.push_back(*promise);
-            std::iter_swap(promise, sentinel);
+    while (promiseIterator != temporaryFilePromises.end()) {
+        if (promiseIterator->use_count() == 1) {
+            promisesToClean.push_back(*promiseIterator);
+            std::iter_swap(promiseIterator, sentinel);
             sentinel--;
             temporaryFilePromises.pop_back();
         } else {
-            promise++;
+            promiseIterator++;
         }
     }
 }
@@ -267,7 +277,7 @@ uint32_t sfz::FilePool::getPreloadSize() const noexcept
     return preloadSize;
 }
 
-void sfz::FilePool::emptyFileLoadingQueue() noexcept
+void sfz::FilePool::emptyFileLoadingQueues() noexcept
 {
     emptyQueue = true;
     while (emptyQueue)
diff --git a/src/sfizz/FilePool.h b/src/sfizz/FilePool.h
index 2d4b251e..a052200c 100644
--- a/src/sfizz/FilePool.h
+++ b/src/sfizz/FilePool.h
@@ -31,7 +31,7 @@
 #include <absl/container/flat_hash_map.h>
 #include <absl/types/optional.h>
 #include "absl/strings/string_view.h"
-#include "moodycamel/concurrentqueue.h"
+#include "moodycamel/blockingconcurrentqueue.h"
 #include <thread>
 #include <sndfile.hh>
 
@@ -77,12 +77,17 @@ using FilePromisePtr = std::shared_ptr<FilePromise>;
 
 class FilePool {
 public:
-    FilePool() { }
+    FilePool()
+    {
+        for (int i = 0; i < config::numBackgroundThreads; ++i)
+            fileLoadingThreadPool.emplace_back( &FilePool::loadingThread, this );
+    }
 
     ~FilePool()
     {
         quitThread = true;
-        fileLoadingThread.join();
+        for (auto& thread: fileLoadingThreadPool)
+            thread.join();
     }
     /**
      * @brief Set the root directory from which to search for files to load
@@ -136,12 +141,13 @@ public:
     uint32_t getPreloadSize() const noexcept;
     void setOversamplingFactor(Oversampling factor) noexcept;
     Oversampling getOversamplingFactor() const noexcept;
-    void emptyFileLoadingQueue() noexcept;
+    void emptyFileLoadingQueues() noexcept;
 private:
     fs::path rootDirectory;
     void loadingThread() noexcept;
 
-    moodycamel::ConcurrentQueue<FilePromisePtr> promiseQueue;
+    moodycamel::BlockingConcurrentQueue<FilePromisePtr> promiseQueue { config::maxVoices };
+    moodycamel::BlockingConcurrentQueue<FilePromisePtr> filledPromiseQueue { config::maxVoices };
     uint32_t preloadSize { config::preloadSize };
     Oversampling oversamplingFactor { config::defaultOversamplingFactor };
     // Signals
@@ -151,7 +157,7 @@ private:
     std::vector<FilePromisePtr> temporaryFilePromises;
     std::vector<FilePromisePtr> promisesToClean;
     absl::flat_hash_map<absl::string_view, PreloadedFileHandle> preloadedFiles;
-    std::thread fileLoadingThread { &FilePool::loadingThread, this };
+    std::vector<std::thread> fileLoadingThreadPool { };
     LEAK_DETECTOR(FilePool);
 };
 }
diff --git a/src/sfizz/Synth.cpp b/src/sfizz/Synth.cpp
index d54aae29..bc3a0b07 100644
--- a/src/sfizz/Synth.cpp
+++ b/src/sfizz/Synth.cpp
@@ -551,7 +551,7 @@ void sfz::Synth::setOversamplingFactor(sfz::Oversampling factor) noexcept
     for (auto& voice: voices)
         voice->reset();
 
-    resources.filePool.emptyFileLoadingQueue();
+    resources.filePool.emptyFileLoadingQueues();
     resources.filePool.setOversamplingFactor(factor);
     oversamplingFactor = factor;
 }