diff --git a/src/audio/SDL_audio.c b/src/audio/SDL_audio.c
index af9175333..eafeb337a 100644
--- a/src/audio/SDL_audio.c
+++ b/src/audio/SDL_audio.c
@@ -400,11 +400,14 @@ mark_device_removed(void *handle, SDL_AudioDeviceItem *devices, SDL_bool *remove
 
 /* The audio backends call this when a device is removed from the system. */
 void
-SDL_RemoveAudioDevice(void *handle)
+SDL_RemoveAudioDevice(const int iscapture, void *handle)
 {
     SDL_LockMutex(current_audio.detectionLock);
-    mark_device_removed(handle, current_audio.inputDevices, &current_audio.captureDevicesRemoved);
-    mark_device_removed(handle, current_audio.outputDevices, &current_audio.outputDevicesRemoved);
+    if (iscapture) {
+        mark_device_removed(handle, current_audio.inputDevices, &current_audio.captureDevicesRemoved);
+    } else {
+        mark_device_removed(handle, current_audio.outputDevices, &current_audio.outputDevicesRemoved);
+    }
     SDL_UnlockMutex(current_audio.detectionLock);
     current_audio.impl.FreeDeviceHandle(handle);
 }
diff --git a/src/audio/SDL_sysaudio.h b/src/audio/SDL_sysaudio.h
index a339f8166..2b4e95688 100644
--- a/src/audio/SDL_sysaudio.h
+++ b/src/audio/SDL_sysaudio.h
@@ -37,7 +37,7 @@ extern void SDL_AddAudioDevice(const int iscapture, const char *name, void *hand
 
 /* Audio targets should call this as devices are removed, so SDL can update
    its list of available devices. */
-extern void SDL_RemoveAudioDevice(void *handle);
+extern void SDL_RemoveAudioDevice(const int iscapture, void *handle);
 
 /* Audio targets should call this if an opened audio device is lost while
    being used. This can happen due to i/o errors, or a device being unplugged,
diff --git a/src/audio/coreaudio/SDL_coreaudio.c b/src/audio/coreaudio/SDL_coreaudio.c
index afd2565dd..22210b050 100644
--- a/src/audio/coreaudio/SDL_coreaudio.c
+++ b/src/audio/coreaudio/SDL_coreaudio.c
@@ -246,7 +246,7 @@ reprocess_device_list(const int iscapture, AudioDeviceList **list)
         if (item->alive) {
             prev = item;
         } else {
-            SDL_RemoveAudioDevice((void *) ((size_t) item->devid));
+            SDL_RemoveAudioDevice(iscapture, (void *) ((size_t) item->devid));
             if (prev) {
                 prev->next = item->next;
             } else {
diff --git a/src/audio/winmm/SDL_winmm.c b/src/audio/winmm/SDL_winmm.c
index bd795cf34..8bc86b8fc 100644
--- a/src/audio/winmm/SDL_winmm.c
+++ b/src/audio/winmm/SDL_winmm.c
@@ -47,7 +47,7 @@ static void DetectWave##typ##Devs(void) { \
         if (wave##typ##GetDevCaps(i,&caps,sizeof(caps))==MMSYSERR_NOERROR) { \
             char *name = WIN_StringToUTF8(caps.szPname); \
             if (name != NULL) { \
-                SDL_AddAudioDevice((int) iscapture, name, (void *) ((size_t) i|(iscap<<31))); \
+                SDL_AddAudioDevice((int) iscapture, name, (void *) ((size_t) i)); \
                 SDL_free(name); \
             } \
         } \
@@ -232,11 +232,6 @@ WINMM_OpenDevice(_THIS, void *handle, const char *devname, int iscapture)
     if (handle != NULL) {  /* specific device requested? */
         const size_t val = (size_t) handle;
         devId = (UINT) val;
-        if (iscapture) {
-            /* we use the top bit to make value unique vs output indices. */
-            SDL_assert((devId & (1<<31)) != 0);
-            devId &= ~(1<<31);
-        }
     }
 
     /* Initialize all variables that we clean on shutdown */