Re-added a simplified version of SDL_SetWindowShape()

In order to handle mouse click transparency this needs to be implemented inside SDL

src/SDL_properties.c

@@ -348,6 +348,7 @@ int SDL_SetPropertyWithCleanup(SDL_PropertiesID props, const char *name, void *v
 
     property = (SDL_Property *)SDL_calloc(1, sizeof(*property));
     if (!property) {
+        SDL_FreePropertyWithCleanup(NULL, property, NULL, SDL_FALSE);
         return -1;
     }
     property->type = SDL_PROPERTY_TYPE_POINTER;
@@ -374,6 +375,17 @@ int SDL_SetProperty(SDL_PropertiesID props, const char *name, void *value)
     return SDL_PrivateSetProperty(props, name, property);
 }
 
+static void CleanupSurface(void *userdata, void *value)
+{
+    SDL_Surface *surface = (SDL_Surface *)value;
+
+    SDL_DestroySurface(surface);
+}
+
+int SDL_SetSurfaceProperty(SDL_PropertiesID props, const char *name, SDL_Surface *surface)
+{
+    return SDL_SetPropertyWithCleanup(props, name, surface, CleanupSurface, NULL);
+}
 
 int SDL_SetStringProperty(SDL_PropertiesID props, const char *name, const char *value)
 {

src/SDL_properties_c.h

@@ -20,4 +20,5 @@
 */
 
 extern int SDL_InitProperties(void);
+extern int SDL_SetSurfaceProperty(SDL_PropertiesID props, const char *name, SDL_Surface *surface);
 extern void SDL_QuitProperties(void);

src/dynapi/SDL_dynapi.sym

@@ -970,6 +970,7 @@ SDL3_0.0.0 {
     SDL_SetRenderColorScale;
     SDL_GetRenderColorScale;
     SDL_RenderGeometryRawFloat;
+    SDL_SetWindowShape;
     # extra symbols go here (don't modify this line)
   local: *;
 };

src/dynapi/SDL_dynapi_overrides.h

@@ -995,3 +995,4 @@
 #define SDL_SetRenderColorScale SDL_SetRenderColorScale_REAL
 #define SDL_GetRenderColorScale SDL_GetRenderColorScale_REAL
 #define SDL_RenderGeometryRawFloat SDL_RenderGeometryRawFloat_REAL
+#define SDL_SetWindowShape SDL_SetWindowShape_REAL

src/dynapi/SDL_dynapi_procs.h

@@ -1020,3 +1020,4 @@ SDL_DYNAPI_PROC(int,SDL_CopyProperties,(SDL_PropertiesID a, SDL_PropertiesID b),
 SDL_DYNAPI_PROC(int,SDL_SetRenderColorScale,(SDL_Renderer *a, float b),(a,b),return)
 SDL_DYNAPI_PROC(int,SDL_GetRenderColorScale,(SDL_Renderer *a, float *b),(a,b),return)
 SDL_DYNAPI_PROC(int,SDL_RenderGeometryRawFloat,(SDL_Renderer *a, SDL_Texture *b, const float *c, int d, const SDL_FColor *e, int f, const float *g, int h, int i, const void *j, int k, int l),(a,b,c,d,e,f,g,h,i,j,k,l),return)
+SDL_DYNAPI_PROC(int,SDL_SetWindowShape,(SDL_Window *a, SDL_Surface *b),(a,b),return)

src/render/SDL_render.c

@@ -989,10 +989,14 @@ SDL_Renderer *SDL_CreateRendererWithProperties(SDL_PropertiesID props)
 
     renderer->color_scale = 1.0f;
 
-    if (SDL_GetWindowFlags(window) & (SDL_WINDOW_HIDDEN | SDL_WINDOW_MINIMIZED)) {
-        renderer->hidden = SDL_TRUE;
-    } else {
-        renderer->hidden = SDL_FALSE;
+    if (window) {
+        if (SDL_GetWindowFlags(window) & SDL_WINDOW_TRANSPARENT) {
+            renderer->transparent_window = SDL_TRUE;
+        }
+
+        if (SDL_GetWindowFlags(window) & (SDL_WINDOW_HIDDEN | SDL_WINDOW_MINIMIZED)) {
+            renderer->hidden = SDL_TRUE;
+        }
     }
 
     new_props = SDL_GetRendererProperties(renderer);
@@ -4247,6 +4251,32 @@ SDL_Surface *SDL_RenderReadPixels(SDL_Renderer *renderer, const SDL_Rect *rect)
     return renderer->RenderReadPixels(renderer, &real_rect);
 }
 
+static void SDL_RenderApplyWindowShape(SDL_Renderer *renderer)
+{
+    SDL_Surface *shape = (SDL_Surface *)SDL_GetProperty(SDL_GetWindowProperties(renderer->window), SDL_PROP_WINDOW_SHAPE_POINTER, NULL);
+    if (shape != renderer->shape_surface) {
+        if (renderer->shape_texture) {
+            SDL_DestroyTexture(renderer->shape_texture);
+            renderer->shape_texture = NULL;
+        }
+
+        if (shape) {
+            /* There's nothing we can do if this fails, so just keep on going */
+            renderer->shape_texture = SDL_CreateTextureFromSurface(renderer, shape);
+
+            SDL_SetTextureBlendMode(renderer->shape_texture,
+                SDL_ComposeCustomBlendMode(
+                    SDL_BLENDFACTOR_ZERO, SDL_BLENDFACTOR_SRC_ALPHA, SDL_BLENDOPERATION_ADD,
+                    SDL_BLENDFACTOR_ZERO, SDL_BLENDFACTOR_SRC_ALPHA, SDL_BLENDOPERATION_ADD));
+        }
+        renderer->shape_surface = shape;
+    }
+
+    if (renderer->shape_texture) {
+        SDL_RenderTexture(renderer, renderer->shape_texture, NULL, NULL);
+    }
+}
+
 static void SDL_SimulateRenderVSync(SDL_Renderer *renderer)
 {
     Uint64 now, elapsed;
@@ -4285,6 +4315,10 @@ int SDL_RenderPresent(SDL_Renderer *renderer)
         SDL_RenderLogicalPresentation(renderer);
     }
 
+    if (renderer->transparent_window) {
+        SDL_RenderApplyWindowShape(renderer);
+    }
+
     FlushRenderCommands(renderer); /* time to send everything to the GPU! */
 
 #if DONT_DRAW_WHILE_HIDDEN

src/render/SDL_sysrender.h

@@ -277,6 +277,11 @@ struct SDL_Renderer
     size_t vertex_data_used;
     size_t vertex_data_allocation;
 
+    /* Shaped window support */
+    SDL_bool transparent_window;
+    SDL_Surface *shape_surface;
+    SDL_Texture *shape_texture;
+
     SDL_PropertiesID props;
 
     void *driverdata;

src/video/SDL_sysvideo.h

@@ -253,6 +253,7 @@ struct SDL_VideoDevice
     int (*UpdateWindowFramebuffer)(SDL_VideoDevice *_this, SDL_Window *window, const SDL_Rect *rects, int numrects);
     void (*DestroyWindowFramebuffer)(SDL_VideoDevice *_this, SDL_Window *window);
     void (*OnWindowEnter)(SDL_VideoDevice *_this, SDL_Window *window);
+    int (*UpdateWindowShape)(SDL_VideoDevice *_this, SDL_Window *window, SDL_Surface *shape);
     int (*FlashWindow)(SDL_VideoDevice *_this, SDL_Window *window, SDL_FlashOperation operation);
     int (*SetWindowFocusable)(SDL_VideoDevice *_this, SDL_Window *window, SDL_bool focusable);
     int (*SyncWindow)(SDL_VideoDevice *_this, SDL_Window *window);

src/video/SDL_video.c

@@ -31,6 +31,7 @@
 #include "SDL_video_c.h"
 #include "../events/SDL_events_c.h"
 #include "../SDL_hints_c.h"
+#include "../SDL_properties_c.h"
 #include "../timer/SDL_timer_c.h"
 #include "SDL_video_capture_c.h"
 
@@ -3485,6 +3486,13 @@ void SDL_OnWindowResized(SDL_Window *window)
 {
     SDL_CheckWindowDisplayChanged(window);
     SDL_CheckWindowPixelSizeChanged(window);
+
+    if ((window->flags & SDL_WINDOW_TRANSPARENT) && _this->UpdateWindowShape) {
+        SDL_Surface *surface = (SDL_Surface *)SDL_GetProperty(window->props, SDL_PROP_WINDOW_SHAPE_POINTER, NULL);
+        if (surface) {
+            _this->UpdateWindowShape(_this, window, surface);
+        }
+    }
 }
 
 void SDL_CheckWindowPixelSizeChanged(SDL_Window *window)
@@ -5054,6 +5062,39 @@ int SDL_SetWindowHitTest(SDL_Window *window, SDL_HitTest callback, void *callbac
     return 0;
 }
 
+int SDL_SetWindowShape(SDL_Window *window, SDL_Surface *shape)
+{
+    SDL_PropertiesID props;
+    SDL_Surface *surface;
+
+    CHECK_WINDOW_MAGIC(window, -1);
+
+    if (!(window->flags & SDL_WINDOW_TRANSPARENT)) {
+        return SDL_SetError("Window must be created with SDL_WINDOW_TRANSPARENT");
+    }
+
+    props = SDL_GetWindowProperties(window);
+    if (!props) {
+        return -1;
+    }
+
+    surface = SDL_ConvertSurfaceFormat(shape, SDL_PIXELFORMAT_ARGB32);
+    if (!surface) {
+        return -1;
+    }
+
+    if (SDL_SetSurfaceProperty(props, SDL_PROP_WINDOW_SHAPE_POINTER, surface) < 0) {
+        return -1;
+    }
+
+    if (_this->UpdateWindowShape) {
+        if (_this->UpdateWindowShape(_this, window, surface) < 0) {
+            return -1;
+        }
+    }
+    return 0;
+}
+
 /*
  * Functions used by iOS application delegates
  */

src/video/cocoa/SDL_cocoawindow.h

@@ -74,6 +74,7 @@ typedef enum
 - (BOOL)isMovingOrFocusClickPending;
 - (void)setFocusClickPending:(NSInteger)button;
 - (void)clearFocusClickPending:(NSInteger)button;
+- (void)updateIgnoreMouseState:(NSEvent *)theEvent;
 - (void)setPendingMoveX:(float)x Y:(float)y;
 - (void)windowDidFinishMoving;
 - (void)onMovingOrFocusClickPendingStateCleared;

src/video/cocoa/SDL_cocoawindow.m

@@ -834,6 +834,28 @@ static SDL_bool Cocoa_IsZoomed(SDL_Window *window)
     }
 }
 
+- (void)updateIgnoreMouseState:(NSEvent *)theEvent
+{
+    SDL_Window *window = _data.window;
+    SDL_Surface *shape = (SDL_Surface *)SDL_GetProperty(SDL_GetWindowProperties(window), SDL_PROP_WINDOW_SHAPE_POINTER, NULL);
+    BOOL ignoresMouseEvents = NO;
+
+    if (shape) {
+        NSPoint point = [theEvent locationInWindow];
+		NSRect windowRect = [[_data.nswindow contentView] frame];
+		if (NSMouseInRect(point, windowRect, NO)) {
+			int x = (int)SDL_roundf((point.x / (window->w - 1)) * (shape->w - 1));
+			int y = (int)SDL_roundf(((window->h - point.y) / (window->h - 1)) * (shape->h - 1));
+			Uint8 a;
+
+			if (SDL_ReadSurfacePixel(shape, x, y, NULL, NULL, NULL, &a) < 0 || a == SDL_ALPHA_TRANSPARENT) {
+				ignoresMouseEvents = YES;
+			}
+		}
+    }
+    _data.nswindow.ignoresMouseEvents = ignoresMouseEvents;
+}
+
 - (void)setPendingMoveX:(float)x Y:(float)y
 {
     pendingWindowWarpX = x;
@@ -1555,6 +1577,10 @@ static int Cocoa_SendMouseButtonClicks(SDL_Mouse *mouse, NSEvent *theEvent, SDL_
     mouseID = mouse->mouseID;
     window = _data.window;
 
+    if (window->flags & SDL_WINDOW_TRANSPARENT) {
+        [self updateIgnoreMouseState:theEvent];
+    }
+
     if ([self processHitTest:theEvent]) {
         SDL_SendWindowEvent(window, SDL_EVENT_WINDOW_HIT_TEST, 0, 0);
         return; /* dragging, drop event. */

src/video/dummy/SDL_nullframebuffer.c

@@ -23,16 +23,11 @@
 #ifdef SDL_VIDEO_DRIVER_DUMMY
 
 #include "../SDL_sysvideo.h"
+#include "../../SDL_properties_c.h"
 #include "SDL_nullframebuffer_c.h"
 
 #define DUMMY_SURFACE "SDL.internal.window.surface"
 
-static void CleanupSurface(void *userdata, void *value)
-{
-    SDL_Surface *surface = (SDL_Surface *)value;
-
-    SDL_DestroySurface(surface);
-}
 
 int SDL_DUMMY_CreateWindowFramebuffer(SDL_VideoDevice *_this, SDL_Window *window, Uint32 *format, void **pixels, int *pitch)
 {
@@ -48,7 +43,7 @@ int SDL_DUMMY_CreateWindowFramebuffer(SDL_VideoDevice *_this, SDL_Window *window
     }
 
     /* Save the info and return! */
-    SDL_SetPropertyWithCleanup(SDL_GetWindowProperties(window), DUMMY_SURFACE, surface, CleanupSurface, NULL);
+    SDL_SetSurfaceProperty(SDL_GetWindowProperties(window), DUMMY_SURFACE, surface);
     *format = surface_format;
     *pixels = surface->pixels;
     *pitch = surface->pitch;

src/video/n3ds/SDL_n3dsframebuffer.c

@@ -23,6 +23,7 @@
 #ifdef SDL_VIDEO_DRIVER_N3DS
 
 #include "../SDL_sysvideo.h"
+#include "../../SDL_properties_c.h"
 #include "SDL_n3dsframebuffer_c.h"
 #include "SDL_n3dsvideo.h"
 
@@ -38,12 +39,6 @@ static int GetDestOffset(int x, int y, int dest_width);
 static int GetSourceOffset(int x, int y, int source_width);
 static void FlushN3DSBuffer(const void *buffer, u32 bufsize, gfxScreen_t screen);
 
-static void CleanupSurface(void *userdata, void *value)
-{
-    SDL_Surface *surface = (SDL_Surface *)value;
-
-    SDL_DestroySurface(surface);
-}
 
 int SDL_N3DS_CreateWindowFramebuffer(SDL_VideoDevice *_this, SDL_Window *window, Uint32 *format, void **pixels, int *pitch)
 {
@@ -57,7 +52,7 @@ int SDL_N3DS_CreateWindowFramebuffer(SDL_VideoDevice *_this, SDL_Window *window,
         return -1;
     }
 
-    SDL_SetPropertyWithCleanup(SDL_GetWindowProperties(window), N3DS_SURFACE, framebuffer, CleanupSurface, NULL);
+    SDL_SetSurfaceProperty(SDL_GetWindowProperties(window), N3DS_SURFACE, framebuffer);
     *format = FRAMEBUFFER_FORMAT;
     *pixels = framebuffer->pixels;
     *pitch = framebuffer->pitch;

src/video/offscreen/SDL_offscreenframebuffer.c

@@ -23,16 +23,11 @@
 #ifdef SDL_VIDEO_DRIVER_OFFSCREEN
 
 #include "../SDL_sysvideo.h"
+#include "../../SDL_properties_c.h"
 #include "SDL_offscreenframebuffer_c.h"
 
 #define OFFSCREEN_SURFACE "SDL.internal.window.surface"
 
-static void CleanupSurface(void *userdata, void *value)
-{
-    SDL_Surface *surface = (SDL_Surface *)value;
-
-    SDL_DestroySurface(surface);
-}
 
 int SDL_OFFSCREEN_CreateWindowFramebuffer(SDL_VideoDevice *_this, SDL_Window *window, Uint32 *format, void **pixels, int *pitch)
 {
@@ -48,7 +43,7 @@ int SDL_OFFSCREEN_CreateWindowFramebuffer(SDL_VideoDevice *_this, SDL_Window *wi
     }
 
     /* Save the info and return! */
-    SDL_SetPropertyWithCleanup(SDL_GetWindowProperties(window), OFFSCREEN_SURFACE, surface, CleanupSurface, NULL);
+    SDL_SetSurfaceProperty(SDL_GetWindowProperties(window), OFFSCREEN_SURFACE, surface);
     *format = surface_format;
     *pixels = surface->pixels;
     *pitch = surface->pitch;

src/video/windows/SDL_windowsshape.c

@@ -0,0 +1,124 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2024 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+#include "SDL_internal.h"
+
+#ifdef SDL_VIDEO_DRIVER_WINDOWS
+
+#include "SDL_windowsvideo.h"
+#include "SDL_windowsshape.h"
+
+
+static void AddRegion(HRGN *mask, int x1, int y1, int x2, int y2)
+{
+    HRGN region = CreateRectRgn(x1, y1, x2, y2);
+    if (*mask) {
+        CombineRgn(*mask, *mask, region, RGN_OR);
+        DeleteObject(region);
+    } else {
+        *mask = region;
+    }
+}
+
+static HRGN GenerateSpanListRegion(SDL_Surface *shape, int offset_x, int offset_y)
+{
+    HRGN mask = NULL;
+    int x, y;
+    int span_start = -1;
+
+    for (y = 0; y < shape->h; ++y) {
+        const Uint8 *a = (const Uint8 *)shape->pixels + y * shape->pitch;
+        for (x = 0; x < shape->w; ++x) {
+            if (*a == SDL_ALPHA_TRANSPARENT) {
+                if (span_start != -1) {
+                    AddRegion(&mask, offset_x + span_start, offset_y + y, offset_x + x, offset_y + y + 1);
+                    span_start = -1;
+                }
+            } else {
+                if (span_start == -1) {
+                    span_start = x;
+                }
+            }
+            a += 4;
+        }
+        if (span_start != -1) {
+            /* Add the final span */
+            AddRegion(&mask, offset_x + span_start, offset_y + y, offset_x + x, offset_y + y + 1);
+            span_start = -1;
+        }
+    }
+    return mask;
+}
+
+int WIN_UpdateWindowShape(SDL_VideoDevice *_this, SDL_Window *window, SDL_Surface *shape)
+{
+    SDL_WindowData *data = window->driverdata;
+    HRGN mask = NULL;
+
+    /* Generate a set of spans for the region */
+    if (shape) {
+        SDL_Surface *stretched = NULL;
+        RECT rect;
+
+        if (shape->w != window->w || shape->h != window->h) {
+            stretched = SDL_CreateSurface(window->w, window->h, SDL_PIXELFORMAT_ARGB32);
+            if (!stretched) {
+                return -1;
+            }
+            if (SDL_SoftStretch(shape, NULL, stretched, NULL, SDL_SCALEMODE_LINEAR) < 0) {
+                SDL_DestroySurface(stretched);
+                return -1;
+            }
+            shape = stretched;
+        }
+
+        rect.top = 0;
+        rect.left = 0;
+        rect.bottom = 0;
+        rect.right = 0;
+        if (!(SDL_GetWindowFlags(data->window) & SDL_WINDOW_BORDERLESS)) {
+            WIN_AdjustWindowRectForHWND(data->hwnd, &rect, 0);
+        }
+
+        mask = GenerateSpanListRegion(shape, -rect.left, -rect.top);
+
+        if (!(SDL_GetWindowFlags(data->window) & SDL_WINDOW_BORDERLESS)) {
+            /* Add the window borders */
+            /* top */
+            AddRegion(&mask, 0, 0, -rect.left + shape->w + rect.right + 1, -rect.top + 1);
+            /* left */
+            AddRegion(&mask, 0, -rect.top, -rect.left + 1, -rect.top + shape->h + 1);
+            /* right */
+            AddRegion(&mask, -rect.left + shape->w, -rect.top, -rect.left + shape->w + rect.right + 1, -rect.top + shape->h + 1);
+            /* bottom */
+            AddRegion(&mask, 0, -rect.top + shape->h, -rect.left + shape->w + rect.right + 1, -rect.top + shape->h + rect.bottom + 1);
+        }
+
+        if (stretched) {
+            SDL_DestroySurface(stretched);
+        }
+    }
+    if (!SetWindowRgn(data->hwnd, mask, TRUE)) {
+        return WIN_SetError("SetWindowRgn failed");
+    }
+    return 0;
+}
+
+#endif /* SDL_VIDEO_DRIVER_WINDOWS */

src/video/windows/SDL_windowsshape.h

@@ -0,0 +1,28 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2024 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+#include "SDL_internal.h"
+
+#ifndef SDL_windowsshape_h_
+#define SDL_windowsshape_h_
+
+extern int WIN_UpdateWindowShape(SDL_VideoDevice *_this, SDL_Window *window, SDL_Surface *shape);
+
+#endif /* SDL_windowsshape_h_ */

src/video/windows/SDL_windowsvideo.c

@@ -210,6 +210,7 @@ static SDL_VideoDevice *WIN_CreateDevice(void)
     device->ShowWindowSystemMenu = WIN_ShowWindowSystemMenu;
     device->SetWindowFocusable = WIN_SetWindowFocusable;
 #endif
+    device->UpdateWindowShape = WIN_UpdateWindowShape;
 
 #ifdef SDL_VIDEO_OPENGL_WGL
     device->GL_LoadLibrary = WIN_GL_LoadLibrary;

src/video/windows/SDL_windowsvideo.h

@@ -42,6 +42,7 @@
 #include "SDL_windowsclipboard.h"
 #include "SDL_windowsevents.h"
 #include "SDL_windowsopengl.h"
+#include "SDL_windowsshape.h"
 
 #if !defined(SDL_PLATFORM_XBOXONE) && !defined(SDL_PLATFORM_XBOXSERIES)
 #include "SDL_windowskeyboard.h"

src/video/x11/SDL_x11shape.c

@@ -0,0 +1,110 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2024 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+#include "SDL_internal.h"
+
+#ifdef SDL_VIDEO_DRIVER_X11
+
+#include "SDL_x11video.h"
+#include "SDL_x11shape.h"
+
+
+#ifdef SDL_VIDEO_DRIVER_X11_XSHAPE
+static Uint8 *GenerateShapeMask(SDL_Surface *shape)
+{
+    int x, y;
+    const size_t ppb = 8;
+    const size_t bytes_per_scanline = (size_t)(shape->w + (ppb - 1)) / ppb;
+    const Uint8 *a;
+    Uint8 *mask;
+    Uint8 *mask_scanline;
+    Uint8 mask_value;
+
+    mask = (Uint8 *)SDL_calloc(1, shape->h * bytes_per_scanline);
+    if (mask) {
+        for (y = 0; y < shape->h; y++) {
+            a = (const Uint8 *)shape->pixels + y * shape->pitch;
+            mask_scanline = mask + y * bytes_per_scanline;
+            for (x = 0; x < shape->w; x++) {
+                mask_value = (*a == SDL_ALPHA_TRANSPARENT) ? 0 : 1;
+                mask_scanline[x / ppb] |= mask_value << (x % ppb);
+                a += 4;
+            }
+        }
+    }
+    return mask;
+}
+#endif /* SDL_VIDEO_DRIVER_X11_XSHAPE */
+
+int X11_UpdateWindowShape(SDL_VideoDevice *_this, SDL_Window *window, SDL_Surface *shape)
+{
+#ifdef SDL_VIDEO_DRIVER_X11_XSHAPE
+    SDL_WindowData *windowdata = window->driverdata;
+    int result = -1;
+
+    /* Generate a set of spans for the region */
+    if (shape) {
+        SDL_Surface *stretched = NULL;
+        Uint8 *mask;
+        Pixmap pixmap;
+
+        if (shape->w != window->w || shape->h != window->h) {
+            stretched = SDL_CreateSurface(window->w, window->h, SDL_PIXELFORMAT_ARGB32);
+            if (!stretched) {
+                return -1;
+            }
+            if (SDL_SoftStretch(shape, NULL, stretched, NULL, SDL_SCALEMODE_LINEAR) < 0) {
+                SDL_DestroySurface(stretched);
+                return -1;
+            }
+            shape = stretched;
+        }
+
+        mask = GenerateShapeMask(shape);
+        if (mask) {
+            pixmap = X11_XCreateBitmapFromData(windowdata->videodata->display, windowdata->xwindow, (const char *)mask, shape->w, shape->h);
+            X11_XShapeCombineMask(windowdata->videodata->display, windowdata->xwindow, ShapeInput, 0, 0, pixmap, ShapeSet);
+            SDL_free(mask);
+
+            result = 0;
+        }
+
+        if (stretched) {
+            SDL_DestroySurface(stretched);
+        }
+    } else {
+        Region region = X11_XCreateRegion();
+        XRectangle rect;
+
+        rect.x = 0;
+        rect.y = 0;
+        rect.width = window->w;
+        rect.height = window->h;
+        X11_XUnionRectWithRegion(&rect, region, region);
+        X11_XShapeCombineRegion(windowdata->videodata->display, windowdata->xwindow, ShapeInput, 0, 0, region, ShapeSet);
+        X11_XDestroyRegion(region);
+        result = 0;
+    }
+#endif /* SDL_VIDEO_DRIVER_X11_XSHAPE */
+
+    return result;
+}
+
+#endif /* SDL_VIDEO_DRIVER_X11 */

src/video/x11/SDL_x11shape.h

@@ -0,0 +1,28 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2024 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+#include "SDL_internal.h"
+
+#ifndef SDL_x11shape_h_
+#define SDL_x11shape_h_
+
+extern int X11_UpdateWindowShape(SDL_VideoDevice *_this, SDL_Window *window, SDL_Surface *shape);
+
+#endif /* SDL_x11shape_h_ */

src/video/x11/SDL_x11sym.h

@@ -156,6 +156,7 @@ SDL_X11_SYM(int,XQueryTree,(Display* a,Window b,Window* c,Window* d,Window** e,u
 SDL_X11_SYM(Bool,XSupportsLocale,(void),(),return)
 SDL_X11_SYM(Status,XmbTextListToTextProperty,(Display* a,char** b,int c,XICCEncodingStyle d,XTextProperty* e),(a,b,c,d,e),return)
 SDL_X11_SYM(Region,XCreateRegion,(void),(),return)
+SDL_X11_SYM(int,XUnionRectWithRegion,(XRectangle *a, Region b, Region c),(a,b,c), return)
 SDL_X11_SYM(void,XDestroyRegion,(Region),(a),)
 SDL_X11_SYM(void,XrmInitialize,(),(),)
 SDL_X11_SYM(char*,XResourceManagerString,(Display *display),(display),)

src/video/x11/SDL_x11video.c

@@ -35,6 +35,7 @@
 #include "SDL_x11xfixes.h"
 #include "SDL_x11xinput2.h"
 #include "SDL_x11messagebox.h"
+#include "SDL_x11shape.h"
 
 #ifdef SDL_VIDEO_OPENGL_EGL
 #include "SDL_x11opengles.h"
@@ -208,6 +209,7 @@ static SDL_VideoDevice *X11_CreateDevice(void)
     device->SetWindowHitTest = X11_SetWindowHitTest;
     device->AcceptDragAndDrop = X11_AcceptDragAndDrop;
     device->FlashWindow = X11_FlashWindow;
+    device->UpdateWindowShape = X11_UpdateWindowShape;
     device->ShowWindowSystemMenu = X11_ShowWindowSystemMenu;
     device->SetWindowFocusable = X11_SetWindowFocusable;
     device->SyncWindow = X11_SyncWindow;