Infinite loop when removing an element from a GtkPaned

In some cases, when removing something from a GtkPaned, the code enters in an infinite loop, showing the message

Gtk-CRITICAL **: 00:22:19.677: gtk_widget_get_parent: assertion ‘GTK_IS_WIDGET (widget)’ failed

over and over again, and the application becomes irresponsible.

A piece of code that triggers this (self-compilable, just save it as test_gtk.c, give it execution permissions, and run it):

#!/bin/bash
cat $0 | sed -nr '/\/\*\*/,$p' | gcc -x c -o ${0::-2} - `pkg-config --cflags --libs gtk4`
${0::-2}
exit 0
/** Here begins the C code */

#include <glib-object.h>
#include <gtk/gtk.h>

GtkApplication *application;
GtkWidget *window, *paned, *label1, *label2, *button, *container, *box1, *box2, *button1, *button2;

void
swap_elements (GObject *caller,
               gpointer  data)
{
    GtkWidget *w1, *w2;
    GtkButton *button = GTK_BUTTON (caller);
    if (caller == button1)
        gtk_box_remove (GTK_BOX (box1), GTK_WIDGET (caller));
    if (caller == button2)
        gtk_box_remove (GTK_BOX (box2), GTK_WIDGET (caller));
    g_print ("Removing the elements from inside the paned\n");
    gtk_paned_set_start_child (GTK_PANED (paned), NULL);
    gtk_paned_set_end_child (GTK_PANED (paned), NULL);
}

void activate_function ()
{
    window = gtk_application_window_new (application);
    gtk_window_set_resizable (GTK_WINDOW (window), TRUE);


    label1 = g_object_ref_sink (gtk_label_new ("This is LABEL1"));
    label2 = g_object_ref_sink (gtk_label_new ("This is LABEL2"));
    paned = gtk_paned_new (GTK_ORIENTATION_HORIZONTAL);
    box1 = g_object_ref_sink (gtk_box_new (GTK_ORIENTATION_VERTICAL, 0));
    box2 = g_object_ref_sink (gtk_box_new (GTK_ORIENTATION_VERTICAL, 0));
    button1 = g_object_ref_sink (gtk_button_new_with_label ("Click here to remove this element from the paned"));
    button2 = g_object_ref_sink (gtk_button_new_with_label ("Click here to remove this element from the paned"));
    gtk_box_append (GTK_BOX (box1), button1);
    gtk_box_append (GTK_BOX (box1), label1);
    gtk_box_append (GTK_BOX (box2), button2);
    gtk_box_append (GTK_BOX (box2), label2);

    gtk_paned_set_start_child (GTK_PANED (paned), GTK_WIDGET (box1));
    gtk_paned_set_end_child (GTK_PANED (paned), GTK_WIDGET (box2));

    g_signal_connect (G_OBJECT (button1), "clicked", G_CALLBACK (swap_elements), NULL);
    g_signal_connect (G_OBJECT (button2), "clicked", G_CALLBACK (swap_elements), NULL);

    gtk_window_set_child (GTK_WINDOW (window), GTK_WIDGET (paned));
    gtk_widget_show (window);
}

int main(int argc, char **argv)
{
    application = gtk_application_new ("com.rastersoft.tiledtest", G_APPLICATION_FLAGS_NONE);
    g_signal_connect (G_OBJECT (application), "activate", G_CALLBACK (activate_function), NULL);
    g_application_run (G_APPLICATION (application), argc, argv);
}