camera: Massive code reworking.

- Simplified public API, simplified backend interface.
- Camera device hotplug events.
- Thread code is split up so it backends that provide own threads can use it.
- Added "dummy" backend.

Note that CoreMedia (Apple) and Android backends need to be updated, as does
the testcamera app (testcameraminimal works).

src/camera/SDL_camera_c.h

@@ -29,4 +29,7 @@ int SDL_CameraInit(const char *driver_name);
 // Shutdown the camera subsystem
 void SDL_QuitCamera(void);
 
+// "Pump" the event queue.
+extern void SDL_UpdateCamera(void);
+
 #endif // SDL_camera_c_h_

src/camera/SDL_syscamera.h

@@ -23,67 +23,141 @@
 #ifndef SDL_syscamera_h_
 #define SDL_syscamera_h_
 
-#include "../SDL_list.h"
+#include "../SDL_hashtable.h"
 
 #define DEBUG_CAMERA 1
 
-// The SDL camera driver
+
+// !!! FIXME: update these drivers!
+#ifdef SDL_CAMERA_DRIVER_COREMEDIA
+#undef SDL_CAMERA_DRIVER_COREMEDIA
+#endif
+#ifdef SDL_CAMERA_DRIVER_ANDROID
+#undef SDL_CAMERA_DRIVER_ANDROID
+#endif
+
 typedef struct SDL_CameraDevice SDL_CameraDevice;
 
+/* Backends should call this as devices are added to the system (such as
+   a USB camera being plugged in), and should also be called for
+   for every device found during DetectDevices(). */
+extern SDL_CameraDevice *SDL_AddCameraDevice(const char *name, int num_specs, const SDL_CameraSpec *specs, void *handle);
+
+/* Backends should call this if an opened camera device is lost.
+   This can happen due to i/o errors, or a device being unplugged, etc. */
+extern void SDL_CameraDeviceDisconnected(SDL_CameraDevice *device);
+
+// Find an SDL_CameraDevice, selected by a callback. NULL if not found. DOES NOT LOCK THE DEVICE.
+extern SDL_CameraDevice *SDL_FindPhysicalCameraDeviceByCallback(SDL_bool (*callback)(SDL_CameraDevice *device, void *userdata), void *userdata);
+
+// These functions are the heart of the camera threads. Backends can call them directly if they aren't using the SDL-provided thread.
+extern void SDL_CameraThreadSetup(SDL_CameraDevice *device);
+extern SDL_bool SDL_CameraThreadIterate(SDL_CameraDevice *device);
+extern void SDL_CameraThreadShutdown(SDL_CameraDevice *device);
+
+typedef struct SurfaceList
+{
+    SDL_Surface *surface;
+    Uint64 timestampNS;
+    struct SurfaceList *next;
+} SurfaceList;
+
 // Define the SDL camera driver structure
 struct SDL_CameraDevice
 {
-    // The device's current camera specification
+    // A mutex for locking
+    SDL_Mutex *lock;
+
+    // Human-readable device name.
+    char *name;
+
+    // When refcount hits zero, we destroy the device object.
+    SDL_AtomicInt refcount;
+
+    // All supported formats/dimensions for this device.
+    SDL_CameraSpec *all_specs;
+
+    // Elements in all_specs.
+    int num_specs;
+
+    // The device's actual specification that the camera is outputting, before conversion.
+    SDL_CameraSpec actual_spec;
+
+    // The device's current camera specification, after conversions.
     SDL_CameraSpec spec;
 
-    // Device name
-    char *dev_name;
+    // Unique value assigned at creation time.
+    SDL_CameraDeviceID instance_id;
+
+    // Driver-specific hardware data on how to open device (`hidden` is driver-specific data _when opened_).
+    void *handle;
+
+    // Pixel data flows from the driver into these, then gets converted for the app if necessary.
+    SDL_Surface *acquire_surface;
+
+    // acquire_surface converts or scales to this surface before landing in output_surfaces, if necessary.
+    SDL_Surface *conversion_surface;
+
+    // A queue of surfaces that buffer converted/scaled frames of video until the app claims them.
+    SurfaceList output_surfaces[8];
+    SurfaceList filled_output_surfaces;        // this is FIFO
+    SurfaceList empty_output_surfaces;         // this is LIFO
+    SurfaceList app_held_output_surfaces;
+
+    // non-zero if acquire_surface needs to be scaled for final output.
+    int needs_scaling;  // -1: downscale, 0: no scaling, 1: upscale
+
+    // SDL_TRUE if acquire_surface needs to be converted for final output.
+    SDL_bool needs_conversion;
 
     // Current state flags
     SDL_AtomicInt shutdown;
-    SDL_AtomicInt enabled;
-    SDL_bool is_spec_set;
-
-    // A mutex for locking the queue buffers
-    SDL_Mutex *device_lock;
-    SDL_Mutex *acquiring_lock;
+    SDL_AtomicInt zombie;
 
     // A thread to feed the camera device
     SDL_Thread *thread;
-    SDL_ThreadID threadid;
 
-    // Queued buffers (if app not using callback).
-    SDL_ListNode *buffer_queue;
+    // Optional properties.
+    SDL_PropertiesID props;
 
-    // Data private to this driver
+    // Data private to this driver, used when device is opened and running.
     struct SDL_PrivateCameraData *hidden;
 };
 
 typedef struct SDL_CameraDriverImpl
 {
     void (*DetectDevices)(void);
-    int (*OpenDevice)(SDL_CameraDevice *_this);
-    void (*CloseDevice)(SDL_CameraDevice *_this);
-    int (*InitDevice)(SDL_CameraDevice *_this);
-    int (*GetDeviceSpec)(SDL_CameraDevice *_this, SDL_CameraSpec *spec);
-    int (*StartCamera)(SDL_CameraDevice *_this);
-    int (*StopCamera)(SDL_CameraDevice *_this);
-    int (*AcquireFrame)(SDL_CameraDevice *_this, SDL_CameraFrame *frame);
-    int (*ReleaseFrame)(SDL_CameraDevice *_this, SDL_CameraFrame *frame);
-    int (*GetNumFormats)(SDL_CameraDevice *_this);
-    int (*GetFormat)(SDL_CameraDevice *_this, int index, Uint32 *format);
-    int (*GetNumFrameSizes)(SDL_CameraDevice *_this, Uint32 format);
-    int (*GetFrameSize)(SDL_CameraDevice *_this, Uint32 format, int index, int *width, int *height);
-    int (*GetDeviceName)(SDL_CameraDeviceID instance_id, char *buf, int size);
-    SDL_CameraDeviceID *(*GetDevices)(int *count);
+    int (*OpenDevice)(SDL_CameraDevice *device, const SDL_CameraSpec *spec);
+    void (*CloseDevice)(SDL_CameraDevice *device);
+    int (*WaitDevice)(SDL_CameraDevice *device);
+    int (*AcquireFrame)(SDL_CameraDevice *device, SDL_Surface *frame, Uint64 *timestampNS); // set frame->pixels, frame->pitch, and *timestampNS!
+    void (*ReleaseFrame)(SDL_CameraDevice *device, SDL_Surface *frame); // Reclaim frame->pixels and frame->pitch!
+    void (*FreeDeviceHandle)(SDL_CameraDevice *device); // SDL is done with this device; free the handle from SDL_AddCameraDevice()
     void (*Deinitialize)(void);
+
+    SDL_bool ProvidesOwnCallbackThread;
 } SDL_CameraDriverImpl;
 
+typedef struct SDL_PendingCameraDeviceEvent
+{
+    Uint32 type;
+    SDL_CameraDeviceID devid;
+    struct SDL_PendingCameraDeviceEvent *next;
+} SDL_PendingCameraDeviceEvent;
+
 typedef struct SDL_CameraDriver
 {
     const char *name;  // The name of this camera driver
     const char *desc;  // The description of this camera driver
     SDL_CameraDriverImpl impl; // the backend's interface
+
+    SDL_RWLock *device_hash_lock;  // A rwlock that protects `device_hash`
+    SDL_HashTable *device_hash;  // the collection of currently-available camera devices
+    SDL_PendingCameraDeviceEvent pending_events;
+    SDL_PendingCameraDeviceEvent *pending_events_tail;
+
+    SDL_AtomicInt device_count;
+    SDL_AtomicInt shutting_down;  // non-zero during SDL_Quit, so we known not to accept any last-minute device hotplugs.
 } SDL_CameraDriver;
 
 typedef struct CameraBootStrap

src/camera/coremedia/SDL_camera_coremedia.m

@@ -135,7 +135,9 @@ static Uint32 nsfourcc_to_sdlformat(NSString *nsfourcc)
     if (SDL_strcmp("yuvs", str) == 0)  return SDL_PIXELFORMAT_UYVY;
     if (SDL_strcmp("420f", str) == 0)  return SDL_PIXELFORMAT_UNKNOWN;
 
-    SDL_Log("Unknown format '%s'", str);
+    #if DEBUG_CAMERA
+    SDL_Log("CAMERA: Unknown format '%s'", str);
+    #endif
 
     return SDL_PIXELFORMAT_UNKNOWN;
 }
@@ -177,8 +179,9 @@ static NSString *sdlformat_to_nsfourcc(Uint32 fmt)
     - (void)captureOutput:(AVCaptureOutput *)output
         didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer
         fromConnection:(AVCaptureConnection *)connection {
-            // !!! FIXME #if DEBUG_CAMERA
-            SDL_Log("Drop frame..");
+            #if DEBUG_CAMERA
+            SDL_Log("CAMERA: Drop frame..");
+            #endif
         }
 @end
 
@@ -362,13 +365,13 @@ static int COREMEDIA_AcquireFrame(SDL_CameraDevice *_this, SDL_CameraFrame *fram
         const int numPlanes = CVPixelBufferGetPlaneCount(image);
         const int planar = CVPixelBufferIsPlanar(image);
 
-#if 0
+        #if DEBUG_CAMERA
         const int w = CVPixelBufferGetWidth(image);
         const int h = CVPixelBufferGetHeight(image);
         const int sz = CVPixelBufferGetDataSize(image);
         const int pitch = CVPixelBufferGetBytesPerRow(image);
-        SDL_Log("buffer planar=%d count:%d %d x %d sz=%d pitch=%d", planar, numPlanes, w, h, sz, pitch);
-#endif
+        SDL_Log("CAMERA: buffer planar=%d count:%d %d x %d sz=%d pitch=%d", planar, numPlanes, w, h, sz, pitch);
+        #endif
 
         CVPixelBufferLockBaseAddress(image, 0);
 

src/camera/dummy/SDL_camera_dummy.c

@@ -0,0 +1,80 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2023 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+#include "SDL_internal.h"
+
+#ifdef SDL_CAMERA_DRIVER_DUMMY
+
+#include "../SDL_syscamera.h"
+
+static int DUMMYCAMERA_OpenDevice(SDL_CameraDevice *device, const SDL_CameraSpec *spec)
+{
+    return SDL_Unsupported();
+}
+
+static void DUMMYCAMERA_CloseDevice(SDL_CameraDevice *device)
+{
+}
+
+static int DUMMYCAMERA_WaitDevice(SDL_CameraDevice *device)
+{
+    return SDL_Unsupported();
+}
+
+static int DUMMYCAMERA_AcquireFrame(SDL_CameraDevice *device, SDL_Surface *frame, Uint64 *timestampNS)
+{
+    return SDL_Unsupported();
+}
+
+static void DUMMYCAMERA_ReleaseFrame(SDL_CameraDevice *device, SDL_Surface *frame)
+{
+}
+
+static void DUMMYCAMERA_DetectDevices(void)
+{
+}
+
+static void DUMMYCAMERA_FreeDeviceHandle(SDL_CameraDevice *device)
+{
+}
+
+static void DUMMYCAMERA_Deinitialize(void)
+{
+}
+
+static SDL_bool DUMMYCAMERA_Init(SDL_CameraDriverImpl *impl)
+{
+    impl->DetectDevices = DUMMYCAMERA_DetectDevices;
+    impl->OpenDevice = DUMMYCAMERA_OpenDevice;
+    impl->CloseDevice = DUMMYCAMERA_CloseDevice;
+    impl->WaitDevice = DUMMYCAMERA_WaitDevice;
+    impl->AcquireFrame = DUMMYCAMERA_AcquireFrame;
+    impl->ReleaseFrame = DUMMYCAMERA_ReleaseFrame;
+    impl->FreeDeviceHandle = DUMMYCAMERA_FreeDeviceHandle;
+    impl->Deinitialize = DUMMYCAMERA_Deinitialize;
+
+    return SDL_TRUE;
+}
+
+CameraBootStrap DUMMYCAMERA_bootstrap = {
+    "dummy", "SDL dummy camera driver", DUMMYCAMERA_Init, SDL_TRUE
+};
+
+#endif

src/dynapi/SDL_dynapi.sym

@@ -957,21 +957,18 @@ SDL3_0.0.0 {
     SDL_SetWindowShape;
     SDL_RenderViewportSet;
     SDL_HasProperty;
+    SDL_GetNumCameraDrivers;
+    SDL_GetCameraDriver;
+    SDL_GetCurrentCameraDriver;
     SDL_GetCameraDevices;
-    SDL_OpenCamera;
-    SDL_SetCameraSpec;
-    SDL_OpenCameraWithSpec;
+    SDL_GetCameraDeviceSupportedSpecs;
     SDL_GetCameraDeviceName;
+    SDL_OpenCameraDevice;
+    SDL_GetCameraInstanceID;
+    SDL_GetCameraProperties;
     SDL_GetCameraSpec;
-    SDL_GetCameraFormat;
-    SDL_GetNumCameraFormats;
-    SDL_GetCameraFrameSize;
-    SDL_GetNumCameraFrameSizes;
-    SDL_GetCameraStatus;
-    SDL_StartCamera;
     SDL_AcquireCameraFrame;
     SDL_ReleaseCameraFrame;
-    SDL_StopCamera;
     SDL_CloseCamera;
     # extra symbols go here (don't modify this line)
   local: *;

src/dynapi/SDL_dynapi_overrides.h

@@ -982,19 +982,16 @@
 #define SDL_SetWindowShape SDL_SetWindowShape_REAL
 #define SDL_RenderViewportSet SDL_RenderViewportSet_REAL
 #define SDL_HasProperty SDL_HasProperty_REAL
+#define SDL_GetNumCameraDrivers SDL_GetNumCameraDrivers_REAL
+#define SDL_GetCameraDriver SDL_GetCameraDriver_REAL
+#define SDL_GetCurrentCameraDriver SDL_GetCurrentCameraDriver_REAL
 #define SDL_GetCameraDevices SDL_GetCameraDevices_REAL
-#define SDL_OpenCamera SDL_OpenCamera_REAL
-#define SDL_SetCameraSpec SDL_SetCameraSpec_REAL
-#define SDL_OpenCameraWithSpec SDL_OpenCameraWithSpec_REAL
+#define SDL_GetCameraDeviceSupportedSpecs SDL_GetCameraDeviceSupportedSpecs_REAL
 #define SDL_GetCameraDeviceName SDL_GetCameraDeviceName_REAL
+#define SDL_OpenCameraDevice SDL_OpenCameraDevice_REAL
+#define SDL_GetCameraInstanceID SDL_GetCameraInstanceID_REAL
+#define SDL_GetCameraProperties SDL_GetCameraProperties_REAL
 #define SDL_GetCameraSpec SDL_GetCameraSpec_REAL
-#define SDL_GetCameraFormat SDL_GetCameraFormat_REAL
-#define SDL_GetNumCameraFormats SDL_GetNumCameraFormats_REAL
-#define SDL_GetCameraFrameSize SDL_GetCameraFrameSize_REAL
-#define SDL_GetNumCameraFrameSizes SDL_GetNumCameraFrameSizes_REAL
-#define SDL_GetCameraStatus SDL_GetCameraStatus_REAL
-#define SDL_StartCamera SDL_StartCamera_REAL
 #define SDL_AcquireCameraFrame SDL_AcquireCameraFrame_REAL
 #define SDL_ReleaseCameraFrame SDL_ReleaseCameraFrame_REAL
-#define SDL_StopCamera SDL_StopCamera_REAL
 #define SDL_CloseCamera SDL_CloseCamera_REAL

src/dynapi/SDL_dynapi_procs.h

@@ -1007,19 +1007,16 @@ SDL_DYNAPI_PROC(int,SDL_RenderGeometryRawFloat,(SDL_Renderer *a, SDL_Texture *b,
 SDL_DYNAPI_PROC(int,SDL_SetWindowShape,(SDL_Window *a, SDL_Surface *b),(a,b),return)
 SDL_DYNAPI_PROC(SDL_bool,SDL_RenderViewportSet,(SDL_Renderer *a),(a),return)
 SDL_DYNAPI_PROC(SDL_bool,SDL_HasProperty,(SDL_PropertiesID a, const char *b),(a,b),return)
+SDL_DYNAPI_PROC(int,SDL_GetNumCameraDrivers,(void),(),return)
+SDL_DYNAPI_PROC(const char*,SDL_GetCameraDriver,(int a),(a),return)
+SDL_DYNAPI_PROC(const char*,SDL_GetCurrentCameraDriver,(void),(),return)
 SDL_DYNAPI_PROC(SDL_CameraDeviceID*,SDL_GetCameraDevices,(int *a),(a),return)
-SDL_DYNAPI_PROC(SDL_CameraDevice*,SDL_OpenCamera,(SDL_CameraDeviceID a),(a),return)
-SDL_DYNAPI_PROC(int,SDL_SetCameraSpec,(SDL_CameraDevice *a, const SDL_CameraSpec *b, SDL_CameraSpec *c, int d),(a,b,c,d),return)
-SDL_DYNAPI_PROC(SDL_CameraDevice*,SDL_OpenCameraWithSpec,(SDL_CameraDeviceID a, const SDL_CameraSpec *b, SDL_CameraSpec *c, int d),(a,b,c,d),return)
-SDL_DYNAPI_PROC(const char*,SDL_GetCameraDeviceName,(SDL_CameraDeviceID a),(a),return)
-SDL_DYNAPI_PROC(int,SDL_GetCameraSpec,(SDL_CameraDevice *a, SDL_CameraSpec *b),(a,b),return)
-SDL_DYNAPI_PROC(int,SDL_GetCameraFormat,(SDL_CameraDevice *a, int b, Uint32 *c),(a,b,c),return)
-SDL_DYNAPI_PROC(int,SDL_GetNumCameraFormats,(SDL_CameraDevice *a),(a),return)
-SDL_DYNAPI_PROC(int,SDL_GetCameraFrameSize,(SDL_CameraDevice *a, Uint32 b, int c, int *d, int *e),(a,b,c,d,e),return)
-SDL_DYNAPI_PROC(int,SDL_GetNumCameraFrameSizes,(SDL_CameraDevice *a, Uint32 b),(a,b),return)
-SDL_DYNAPI_PROC(SDL_CameraStatus,SDL_GetCameraStatus,(SDL_CameraDevice *a),(a),return)
-SDL_DYNAPI_PROC(int,SDL_StartCamera,(SDL_CameraDevice *a),(a),return)
-SDL_DYNAPI_PROC(int,SDL_AcquireCameraFrame,(SDL_CameraDevice *a, SDL_CameraFrame *b),(a,b),return)
-SDL_DYNAPI_PROC(int,SDL_ReleaseCameraFrame,(SDL_CameraDevice *a, SDL_CameraFrame *b),(a,b),return)
-SDL_DYNAPI_PROC(int,SDL_StopCamera,(SDL_CameraDevice *a),(a),return)
-SDL_DYNAPI_PROC(void,SDL_CloseCamera,(SDL_CameraDevice *a),(a),)
+SDL_DYNAPI_PROC(SDL_CameraSpec*,SDL_GetCameraDeviceSupportedSpecs,(SDL_CameraDeviceID a, int *b),(a,b),return)
+SDL_DYNAPI_PROC(char*,SDL_GetCameraDeviceName,(SDL_CameraDeviceID a),(a),return)
+SDL_DYNAPI_PROC(SDL_Camera*,SDL_OpenCameraDevice,(SDL_CameraDeviceID a, const SDL_CameraSpec *b),(a,b),return)
+SDL_DYNAPI_PROC(SDL_CameraDeviceID,SDL_GetCameraInstanceID,(SDL_Camera *a),(a),return)
+SDL_DYNAPI_PROC(SDL_PropertiesID,SDL_GetCameraProperties,(SDL_Camera *a),(a),return)
+SDL_DYNAPI_PROC(int,SDL_GetCameraSpec,(SDL_Camera *a, SDL_CameraSpec *b),(a,b),return)
+SDL_DYNAPI_PROC(SDL_Surface*,SDL_AcquireCameraFrame,(SDL_Camera *a, Uint64 *b),(a,b),return)
+SDL_DYNAPI_PROC(int,SDL_ReleaseCameraFrame,(SDL_Camera *a, SDL_Surface *b),(a,b),return)
+SDL_DYNAPI_PROC(void,SDL_CloseCamera,(SDL_Camera *a),(a),)

src/events/SDL_events.c

@@ -25,6 +25,7 @@
 #include "SDL_events_c.h"
 #include "../SDL_hints_c.h"
 #include "../audio/SDL_audio_c.h"
+#include "../camera/SDL_camera_c.h"
 #include "../timer/SDL_timer_c.h"
 #ifndef SDL_JOYSTICK_DISABLED
 #include "../joystick/SDL_joystick_c.h"
@@ -554,6 +555,15 @@ static void SDL_LogEvent(const SDL_Event *event)
         break;
 #undef PRINT_AUDIODEV_EVENT
 
+#define PRINT_CAMERADEV_EVENT(event) (void)SDL_snprintf(details, sizeof(details), " (timestamp=%u which=%u)", (uint)event->cdevice.timestamp, (uint)event->cdevice.which)
+        SDL_EVENT_CASE(SDL_EVENT_CAMERA_DEVICE_ADDED)
+        PRINT_CAMERADEV_EVENT(event);
+        break;
+        SDL_EVENT_CASE(SDL_EVENT_CAMERA_DEVICE_REMOVED)
+        PRINT_CAMERADEV_EVENT(event);
+        break;
+#undef PRINT_CAMERADEV_EVENT
+
         SDL_EVENT_CASE(SDL_EVENT_SENSOR_UPDATE)
         (void)SDL_snprintf(details, sizeof(details), " (timestamp=%u which=%d data[0]=%f data[1]=%f data[2]=%f data[3]=%f data[4]=%f data[5]=%f)",
                            (uint)event->sensor.timestamp, (int)event->sensor.which,
@@ -942,6 +952,10 @@ static void SDL_PumpEventsInternal(SDL_bool push_sentinel)
     SDL_UpdateAudio();
 #endif
 
+#ifndef SDL_CAMERA_DISABLED
+    SDL_UpdateCamera();
+#endif
+
 #ifndef SDL_SENSOR_DISABLED
     /* Check for sensor state change */
     if (SDL_update_sensors) {