gitea/SDL3_fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

camera: Emscripten support!

Browse Source 
This also adds code to deal with waiting for the user to approve camera
access, reworks testcameraminimal to use main callbacks, etc.
This commit is contained in:
Ryan C. Gordon
2023-12-22 01:23:49 -05:00
parent 182f707284
commit 67708f9110
14 changed files with 559 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
src/camera/SDL_camera.c
View File

@@ -40,6 +40,9 @@ static const CameraBootStrap *const bootstrap[] = {
#ifdef SDL_CAMERA_DRIVER_ANDROID
&ANDROIDCAMERA_bootstrap,
#endif
#ifdef SDL_CAMERA_DRIVER_EMSCRIPTEN
&EMSCRIPTENCAMERA_bootstrap,
#endif
#ifdef SDL_CAMERA_DRIVER_DUMMY
&DUMMYCAMERA_bootstrap,
#endif
@@ -247,8 +250,8 @@ static int SDLCALL CameraSpecCmp(const void *vpa, const void *vpb)
SDL_CameraDevice *SDL_AddCameraDevice(const char *name, int num_specs, const SDL_CameraSpec *specs, void *handle)
{
SDL_assert(name != NULL);
SDL_assert(num_specs > 0);
SDL_assert(specs != NULL);
SDL_assert(num_specs >= 0);
SDL_assert((specs != NULL) == (num_specs > 0));
SDL_assert(handle != NULL);
SDL_LockRWLockForReading(camera_driver.device_hash_lock);
@@ -284,22 +287,24 @@ SDL_CameraDevice *SDL_AddCameraDevice(const char *name, int num_specs, const SDL
return NULL;
}
SDL_memcpy(device->all_specs, specs, sizeof (*specs) * num_specs);
SDL_qsort(device->all_specs, num_specs, sizeof (*specs), CameraSpecCmp);
if (num_specs > 0) {
SDL_memcpy(device->all_specs, specs, sizeof (*specs) * num_specs);
SDL_qsort(device->all_specs, num_specs, sizeof (*specs), CameraSpecCmp);
// weed out duplicates, just in case.
for (int i = 0; i < num_specs; i++) {
SDL_CameraSpec *a = &device->all_specs[i];
SDL_CameraSpec *b = &device->all_specs[i + 1];
if (SDL_memcmp(a, b, sizeof (*a)) == 0) {
SDL_memmove(a, b, sizeof (*specs) * (num_specs - i));
i--;
num_specs--;
// weed out duplicates, just in case.
for (int i = 0; i < num_specs; i++) {
SDL_CameraSpec *a = &device->all_specs[i];
SDL_CameraSpec *b = &device->all_specs[i + 1];
if (SDL_memcmp(a, b, sizeof (*a)) == 0) {
SDL_memmove(a, b, sizeof (*specs) * (num_specs - i));
i--;
num_specs--;
}
}
}
#if DEBUG_CAMERA
SDL_Log("CAMERA: Adding device ('%s') with %d spec%s:", name, num_specs, (num_specs == 1) ? "" : "s");
SDL_Log("CAMERA: Adding device ('%s') with %d spec%s%s", name, num_specs, (num_specs == 1) ? "" : "s", (num_specs == 0) ? "" : ":");
for (int i = 0; i < num_specs; i++) {
const SDL_CameraSpec *spec = &device->all_specs[i];
SDL_Log("CAMERA: - fmt=%s, w=%d, h=%d, numerator=%d, denominator=%d", SDL_GetPixelFormatName(spec->format), spec->width, spec->height, spec->interval_numerator, spec->interval_denominator);
@@ -398,6 +403,42 @@ sdfsdf
}
}
void SDL_CameraDevicePermissionOutcome(SDL_CameraDevice *device, SDL_bool approved)
{
if (!device) {
return;
}
SDL_PendingCameraDeviceEvent pending;
pending.next = NULL;
SDL_PendingCameraDeviceEvent *pending_tail = &pending;
const int permission = approved ? 1 : -1;
ObtainPhysicalCameraDeviceObj(device);
if (device->permission != permission) {
device->permission = permission;
SDL_PendingCameraDeviceEvent *p = (SDL_PendingCameraDeviceEvent *) SDL_malloc(sizeof (SDL_PendingCameraDeviceEvent));
if (p) { // if this failed, no event for you, but you have deeper problems anyhow.
p->type = approved ? SDL_EVENT_CAMERA_DEVICE_APPROVED : SDL_EVENT_CAMERA_DEVICE_DENIED;
p->devid = device->instance_id;
p->next = NULL;
pending_tail->next = p;
pending_tail = p;
}
}
ReleaseCameraDevice(device);
SDL_LockRWLockForWriting(camera_driver.device_hash_lock);
SDL_assert(camera_driver.pending_events_tail != NULL);
SDL_assert(camera_driver.pending_events_tail->next == NULL);
camera_driver.pending_events_tail->next = pending.next;
camera_driver.pending_events_tail = pending_tail;
SDL_UnlockRWLock(camera_driver.device_hash_lock);
}
SDL_CameraDevice *SDL_FindPhysicalCameraDeviceByCallback(SDL_bool (*callback)(SDL_CameraDevice *device, void *userdata), void *userdata)
{
if (!SDL_GetCurrentCameraDriver()) {
@@ -439,7 +480,14 @@ int SDL_GetCameraFormat(SDL_Camera *camera, SDL_CameraSpec *spec)
}
SDL_CameraDevice *device = (SDL_CameraDevice *) camera; // currently there's no separation between physical and logical device.
SDL_copyp(spec, &device->spec);
ObtainPhysicalCameraDeviceObj(device);
const int retval = (device->permission > 0) ? 0 : SDL_SetError("Camera permission has not been granted");
if (retval == 0) {
SDL_copyp(spec, &device->spec);
} else {
SDL_zerop(spec);
}
ReleaseCameraDevice(device);
return 0;
}
@@ -545,7 +593,11 @@ SDL_bool SDL_CameraThreadIterate(SDL_CameraDevice *device)
return SDL_FALSE; // we're done, shut it down.
}
// !!! FIXME: this should block elsewhere without holding the lock until a frame is available, like the audio subsystem does.
const int permission = device->permission;
if (permission <= 0) {
SDL_UnlockMutex(device->lock);
return (permission < 0) ? SDL_FALSE : SDL_TRUE; // if permission was denied, shut it down. if undecided, we're done for now.
}
SDL_bool failed = SDL_FALSE; // set to true if disaster worthy of treating the device as lost has happened.
SDL_Surface *acquired = NULL;
@@ -661,7 +713,7 @@ static int SDLCALL CameraThread(void *devicep)
SDL_CameraDevice *device = (SDL_CameraDevice *) devicep;
#if DEBUG_CAMERA
SDL_Log("CAMERA: Start thread 'SDL_CameraThread'");
SDL_Log("CAMERA: dev[%p] Start thread 'CameraThread'", devicep);
#endif
SDL_assert(device != NULL);
@@ -676,7 +728,7 @@ static int SDLCALL CameraThread(void *devicep)
SDL_CameraThreadShutdown(device);
#if DEBUG_CAMERA
SDL_Log("CAMERA: dev[%p] End thread 'SDL_CameraThread'", (void *)device);
SDL_Log("CAMERA: dev[%p] End thread 'CameraThread'", devicep);
#endif
return 0;
@@ -697,9 +749,13 @@ static void ChooseBestCameraSpec(SDL_CameraDevice *device, const SDL_CameraSpec
SDL_zerop(closest);
SDL_assert(((Uint32) SDL_PIXELFORMAT_UNKNOWN) == 0); // since we SDL_zerop'd to this value.
if (!spec) { // nothing specifically requested, get the best format we can...
if (device->num_specs == 0) { // device listed no specs! You get whatever you want!
if (spec) {
SDL_copyp(closest, spec);
}
return;
} else if (!spec) { // nothing specifically requested, get the best format we can...
// we sorted this into the "best" format order when adding the camera.
SDL_assert(device->num_specs > 0);
SDL_copyp(closest, &device->all_specs[0]);
} else { // specific thing requested, try to get as close to that as possible...
const int num_specs = device->num_specs;
@@ -924,6 +980,12 @@ SDL_Surface *SDL_AcquireCameraFrame(SDL_Camera *camera, Uint64 *timestampNS)
ObtainPhysicalCameraDeviceObj(device);
if (device->permission <= 0) {
ReleaseCameraDevice(device);
SDL_SetError("Camera permission has not been granted");
return NULL;
}
SDL_Surface *retval = NULL;
// frames are in this list from newest to oldest, so find the end of the list...
@@ -996,8 +1058,6 @@ int SDL_ReleaseCameraFrame(SDL_Camera *camera, SDL_Surface *frame)
return 0;
}
// !!! FIXME: add a way to "pause" camera output.
SDL_CameraDeviceID SDL_GetCameraInstanceID(SDL_Camera *camera)
{
SDL_CameraDeviceID retval = 0;
@@ -1031,6 +1091,22 @@ SDL_PropertiesID SDL_GetCameraProperties(SDL_Camera *camera)
return retval;
}
int SDL_GetCameraPermissionState(SDL_Camera *camera)
{
int retval;
if (!camera) {
retval = SDL_InvalidParamError("camera");
} else {
SDL_CameraDevice *device = (SDL_CameraDevice *) camera; // currently there's no separation between physical and logical device.
ObtainPhysicalCameraDeviceObj(device);
retval = device->permission;
ReleaseCameraDevice(device);
}
return retval;
}
static void CompleteCameraEntryPoints(void)
{
// this doesn't currently fill in stub implementations, it just asserts the backend filled them all in.

7
src/camera/SDL_syscamera.h
View File

@@ -50,6 +50,9 @@ extern void SDL_CameraDeviceDisconnected(SDL_CameraDevice *device);
// Find an SDL_CameraDevice, selected by a callback. NULL if not found. DOES NOT LOCK THE DEVICE.
extern SDL_CameraDevice *SDL_FindPhysicalCameraDeviceByCallback(SDL_bool (*callback)(SDL_CameraDevice *device, void *userdata), void *userdata);
// Backends should call this when the user has approved/denied access to a camera.
extern void SDL_CameraDevicePermissionOutcome(SDL_CameraDevice *device, SDL_bool approved);
// These functions are the heart of the camera threads. Backends can call them directly if they aren't using the SDL-provided thread.
extern void SDL_CameraThreadSetup(SDL_CameraDevice *device);
extern SDL_bool SDL_CameraThreadIterate(SDL_CameraDevice *device);
@@ -129,6 +132,9 @@ struct SDL_CameraDevice
// Optional properties.
SDL_PropertiesID props;
// -1: user denied permission, 0: waiting for user response, 1: user approved permission.
int permission;
// Data private to this driver, used when device is opened and running.
struct SDL_PrivateCameraData *hidden;
};
@@ -182,5 +188,6 @@ extern CameraBootStrap DUMMYCAMERA_bootstrap;
extern CameraBootStrap V4L2_bootstrap;
extern CameraBootStrap COREMEDIA_bootstrap;
extern CameraBootStrap ANDROIDCAMERA_bootstrap;
extern CameraBootStrap EMSCRIPTENCAMERA_bootstrap;
#endif // SDL_syscamera_h_

269
src/camera/emscripten/SDL_camera_emscripten.c Normal file
View File

@@ -0,0 +1,269 @@
/*
Simple DirectMedia Layer
Copyright (C) 1997-2023 Sam Lantinga <slouken@libsdl.org>
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must not
claim that you wrote the original software. If you use this software
in a product, an acknowledgment in the product documentation would be
appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must not be
misrepresented as being the original software.
3. This notice may not be removed or altered from any source distribution.
*/
#include "SDL_internal.h"
#ifdef SDL_CAMERA_DRIVER_EMSCRIPTEN
#include "../SDL_syscamera.h"
#include "../SDL_camera_c.h"
#include "../../video/SDL_pixels_c.h"
#include <emscripten/emscripten.h>
// just turn off clang-format for this whole file, this INDENT_OFF stuff on
// each EM_ASM section is ugly.
/* *INDENT-OFF* */ /* clang-format off */
EM_JS_DEPS(sdlcamera, "$dynCall");
static int EMSCRIPTENCAMERA_WaitDevice(SDL_CameraDevice *device)
{
SDL_assert(!"This shouldn't be called"); // we aren't using SDL's internal thread.
return -1;
}
static int EMSCRIPTENCAMERA_AcquireFrame(SDL_CameraDevice *device, SDL_Surface *frame, Uint64 *timestampNS)
{
void *rgba = SDL_malloc(device->actual_spec.width * device->actual_spec.height * 4);
if (!rgba) {
return SDL_OutOfMemory();
}
*timestampNS = SDL_GetTicksNS(); // best we can do here.
const int rc = MAIN_THREAD_EM_ASM_INT({
const w = $0;
const h = $1;
const rgba = $2;
const SDL3 = Module['SDL3'];
if ((typeof(SDL3) === 'undefined') || (typeof(SDL3.camera) === 'undefined') || (typeof(SDL3.camera.ctx2d) === 'undefined')) {
return 0; // don't have something we need, oh well.
}
SDL3.camera.ctx2d.drawImage(SDL3.camera.video, 0, 0, w, h);
const imgrgba = SDL3.camera.ctx2d.getImageData(0, 0, w, h).data;
Module.HEAPU8.set(imgrgba, rgba);
return 1;
}, device->actual_spec.width, device->actual_spec.height, rgba);
if (!rc) {
SDL_free(rgba);
return 0; // something went wrong, maybe shutting down; just don't return a frame.
}
frame->pixels = rgba;
frame->pitch = device->actual_spec.width * 4;
return 1;
}
static void EMSCRIPTENCAMERA_ReleaseFrame(SDL_CameraDevice *device, SDL_Surface *frame)
{
SDL_free(frame->pixels);
frame->pixels = NULL;
frame->pitch = 0;
}
static void EMSCRIPTENCAMERA_CloseDevice(SDL_CameraDevice *device)
{
if (device) {
MAIN_THREAD_EM_ASM({
const SDL3 = Module['SDL3'];
if ((typeof(SDL3) === 'undefined') || (typeof(SDL3.camera) === 'undefined') || (typeof(SDL3.camera.stream) === 'undefined')) {
return; // camera was closed and/or subsystem was shut down, we're already done.
}
SDL3.camera.stream.getTracks().forEach(track => track.stop()); // stop all recording.
_SDL_free(SDL3.camera.rgba);
SDL3.camera = {}; // dump our references to everything.
});
SDL_free(device->hidden);
device->hidden = NULL;
}
}
static void SDLEmscriptenCameraDevicePermissionOutcome(SDL_CameraDevice *device, int approved, int w, int h, int fps)
{
device->spec.width = device->actual_spec.width = w;
device->spec.height = device->actual_spec.height = h;
device->spec.interval_numerator = device->actual_spec.interval_numerator = 1;
device->spec.interval_denominator = device->actual_spec.interval_denominator = fps;
SDL_CameraDevicePermissionOutcome(device, approved ? SDL_TRUE : SDL_FALSE);
}
static int EMSCRIPTENCAMERA_OpenDevice(SDL_CameraDevice *device, const SDL_CameraSpec *spec)
{
MAIN_THREAD_EM_ASM({
// Since we can't get actual specs until we make a move that prompts the user for
// permission, we don't list any specs for the device and wrangle it during device open.
const device = $0;
const w = $1;
const h = $2;
const interval_numerator = $3;
const interval_denominator = $4;
const outcome = $5;
const iterate = $6;
const constraints = {};
if ((w <= 0) || (h <= 0)) {
constraints.video = true; // didn't ask for anything, let the system choose.
} else {
constraints.video = {}; // asked for a specific thing: request it as "ideal" but take closest hardware will offer.
constraints.video.width = w;
constraints.video.height = h;
}
if ((interval_numerator > 0) && (interval_denominator > 0)) {
var fps = interval_denominator / interval_numerator;
constraints.video.frameRate = { ideal: fps };
}
function grabNextCameraFrame() { // !!! FIXME: this (currently) runs as a requestAnimationFrame callback, for lack of a better option.
const SDL3 = Module['SDL3'];
if ((typeof(SDL3) === 'undefined') || (typeof(SDL3.camera) === 'undefined') || (typeof(SDL3.camera.stream) === 'undefined')) {
return; // camera was closed and/or subsystem was shut down, stop iterating here.
}
// time for a new frame from the camera?
const nextframems = SDL3.camera.next_frame_time;
const now = performance.now();
if (now >= nextframems) {
dynCall('vi', iterate, [device]); // calls SDL_CameraThreadIterate, which will call our AcquireFrame implementation.
// bump ahead but try to stay consistent on timing, in case we dropped frames.
while (SDL3.camera.next_frame_time < now) {
SDL3.camera.next_frame_time += SDL3.camera.fpsincrms;
}
}
requestAnimationFrame(grabNextCameraFrame); // run this function again at the display framerate. (!!! FIXME: would this be better as requestIdleCallback?)
}
navigator.mediaDevices.getUserMedia(constraints)
.then((stream) => {
const settings = stream.getVideoTracks()[0].getSettings();
const actualw = settings.width;
const actualh = settings.height;
const actualfps = settings.frameRate;
console.log("Camera is opened! Actual spec: (" + actualw + "x" + actualh + "), fps=" + actualfps);
dynCall('viiiii', outcome, [device, 1, actualw, actualh, actualfps]);
const video = document.createElement("video");
video.width = actualw;
video.height = actualh;
video.style.display = 'none'; // we need to attach this to a hidden video node so we can read it as pixels.
video.srcObject = stream;
const canvas = document.createElement("canvas");
canvas.width = actualw;
canvas.height = actualh;
canvas.style.display = 'none'; // we need to attach this to a hidden video node so we can read it as pixels.
const ctx2d = canvas.getContext('2d');
const SDL3 = Module['SDL3'];
SDL3.camera.width = actualw;
SDL3.camera.height = actualh;
SDL3.camera.fps = actualfps;
SDL3.camera.fpsincrms = 1000.0 / actualfps;
SDL3.camera.stream = stream;
SDL3.camera.video = video;
SDL3.camera.canvas = canvas;
SDL3.camera.ctx2d = ctx2d;
SDL3.camera.rgba = 0;
SDL3.camera.next_frame_time = performance.now();
video.play();
video.addEventListener('loadedmetadata', () => {
grabNextCameraFrame(); // start this loop going.
});
})
.catch((err) => {
console.error("Tried to open camera but it threw an error! " + err.name + ": " + err.message);
dynCall('viiiii', outcome, [device, 0, 0, 0, 0]); // we call this a permission error, because it probably is.
});
}, device, spec->width, spec->height, spec->interval_numerator, spec->interval_denominator, SDLEmscriptenCameraDevicePermissionOutcome, SDL_CameraThreadIterate);
return 0; // the real work waits until the user approves a camera.
}
static void EMSCRIPTENCAMERA_FreeDeviceHandle(SDL_CameraDevice *device)
{
// no-op.
}
static void EMSCRIPTENCAMERA_Deinitialize(void)
{
MAIN_THREAD_EM_ASM({
if (typeof(Module['SDL3']) !== 'undefined') {
Module['SDL3'].camera = undefined;
}
});
}
static void EMSCRIPTENCAMERA_DetectDevices(void)
{
// `navigator.mediaDevices` is not defined if unsupported or not in a secure context!
const int supported = MAIN_THREAD_EM_ASM_INT({ return (navigator.mediaDevices === undefined) ? 0 : 1; });
// if we have support at all, report a single generic camera with no specs.
// We'll find out if there really _is_ a camera when we try to open it, but querying it for real here
// will pop up a user permission dialog warning them we're trying to access the camera, and we generally
// don't want that during SDL_Init().
if (supported) {
SDL_AddCameraDevice("Web browser's camera", 0, NULL, (void *) (size_t) 0x1);
}
}
static SDL_bool EMSCRIPTENCAMERA_Init(SDL_CameraDriverImpl *impl)
{
SDL_Log("EMSCRIPTENCAMERA_Init, %s:%d", __FILE__, __LINE__);
MAIN_THREAD_EM_ASM({
if (typeof(Module['SDL3']) === 'undefined') {
Module['SDL3'] = {};
}
Module['SDL3'].camera = {};
});
SDL_Log("EMSCRIPTENCAMERA_Init, %s:%d", __FILE__, __LINE__);
impl->DetectDevices = EMSCRIPTENCAMERA_DetectDevices;
impl->OpenDevice = EMSCRIPTENCAMERA_OpenDevice;
impl->CloseDevice = EMSCRIPTENCAMERA_CloseDevice;
impl->WaitDevice = EMSCRIPTENCAMERA_WaitDevice;
impl->AcquireFrame = EMSCRIPTENCAMERA_AcquireFrame;
impl->ReleaseFrame = EMSCRIPTENCAMERA_ReleaseFrame;
impl->FreeDeviceHandle = EMSCRIPTENCAMERA_FreeDeviceHandle;
impl->Deinitialize = EMSCRIPTENCAMERA_Deinitialize;
impl->ProvidesOwnCallbackThread = SDL_TRUE;
return SDL_TRUE;
}
CameraBootStrap EMSCRIPTENCAMERA_bootstrap = {
"emscripten", "SDL Emscripten MediaStream camera driver", EMSCRIPTENCAMERA_Init, SDL_FALSE
};
/* *INDENT-ON* */ /* clang-format on */
#endif // SDL_CAMERA_DRIVER_EMSCRIPTEN

3
src/camera/v4l2/SDL_camera_v4l2.c
View File

@@ -619,6 +619,9 @@ static int V4L2_OpenDevice(SDL_CameraDevice *device, const SDL_CameraSpec *spec)
}
}
// Currently there is no user permission prompt for camera access, but maybe there will be a D-Bus portal interface at some point.
SDL_CameraDevicePermissionOutcome(device, SDL_TRUE);
return 0;
}

1
src/dynapi/SDL_dynapi.sym
View File

@@ -970,6 +970,7 @@ SDL3_0.0.0 {
SDL_AcquireCameraFrame;
SDL_ReleaseCameraFrame;
SDL_CloseCamera;
SDL_GetCameraPermissionState;
# extra symbols go here (don't modify this line)
local: *;
};

1
src/dynapi/SDL_dynapi_overrides.h
View File

@@ -995,3 +995,4 @@
#define SDL_AcquireCameraFrame SDL_AcquireCameraFrame_REAL
#define SDL_ReleaseCameraFrame SDL_ReleaseCameraFrame_REAL
#define SDL_CloseCamera SDL_CloseCamera_REAL
#define SDL_GetCameraPermissionState SDL_GetCameraPermissionState_REAL

1
src/dynapi/SDL_dynapi_procs.h
View File

@@ -1020,3 +1020,4 @@ SDL_DYNAPI_PROC(int,SDL_GetCameraFormat,(SDL_Camera *a, SDL_CameraSpec *b),(a,b)
SDL_DYNAPI_PROC(SDL_Surface*,SDL_AcquireCameraFrame,(SDL_Camera *a, Uint64 *b),(a,b),return)
SDL_DYNAPI_PROC(int,SDL_ReleaseCameraFrame,(SDL_Camera *a, SDL_Surface *b),(a,b),return)
SDL_DYNAPI_PROC(void,SDL_CloseCamera,(SDL_Camera *a),(a),)
SDL_DYNAPI_PROC(int,SDL_GetCameraPermissionState,(SDL_Camera *a),(a),return)

6
src/events/SDL_events.c
View File

@@ -562,6 +562,12 @@ static void SDL_LogEvent(const SDL_Event *event)
SDL_EVENT_CASE(SDL_EVENT_CAMERA_DEVICE_REMOVED)
PRINT_CAMERADEV_EVENT(event);
break;
SDL_EVENT_CASE(SDL_EVENT_CAMERA_DEVICE_APPROVED)
PRINT_CAMERADEV_EVENT(event);
break;
SDL_EVENT_CASE(SDL_EVENT_CAMERA_DEVICE_DENIED)
PRINT_CAMERADEV_EVENT(event);
break;
#undef PRINT_CAMERADEV_EVENT
SDL_EVENT_CASE(SDL_EVENT_SENSOR_UPDATE)