diff --git a/shell/pps-document-view.c b/shell/pps-document-view.c
index e18c5255fc6665108f5fb49d08b959790380e822..1750c4b3938aa46313aefecc5f998744035b317d 100644
--- a/shell/pps-document-view.c
+++ b/shell/pps-document-view.c
@@ -128,7 +128,6 @@ typedef struct
 
 	GMenu *annot_menu;
 	GtkWidget *annot_menu_child;
-	gboolean annot_menu_child_added;
 
 	GSimpleActionGroup *document_action_group;
 
@@ -279,10 +278,18 @@ pps_document_view_set_default_actions (PpsDocumentView *pps_doc_view)
 		                                      FALSE);
 	}
 
-	if (!PPS_IS_DOCUMENT_ANNOTATIONS (document)) {
+	if (PPS_IS_DOCUMENT_ANNOTATIONS (document) &&
+	    pps_document_annotations_can_add_annotation (PPS_DOCUMENT_ANNOTATIONS (document))) {
+		g_autoptr (GMenuItem) item = g_menu_item_new (NULL, NULL);
+		g_menu_item_set_attribute (item, "custom", "s", "palette");
+		g_menu_insert_item (priv->annot_menu, 0, item);
+
+		gtk_popover_menu_add_child (GTK_POPOVER_MENU (priv->view_popup),
+		                            priv->annot_menu_child, "palette");
+	} else {
 		pps_document_view_set_action_enabled (pps_doc_view,
 		                                      "add-text-annotation",
-		                                      pps_document_annotations_can_add_annotation (PPS_DOCUMENT_ANNOTATIONS (document)));
+		                                      FALSE);
 	}
 
 	if (PPS_IS_DOCUMENT_SIGNATURES (document)) {
@@ -649,24 +656,6 @@ view_selection_changed_cb (PpsView *view,
 
 	pps_document_view_set_action_enabled (window, "add-highlight-annotation",
 	                                      can_annotate && has_selection);
-
-	if (can_annotate && has_selection && !priv->annot_menu_child_added) {
-		g_autoptr (GMenuItem) item = g_menu_item_new (NULL, NULL);
-
-		g_menu_item_set_attribute (item, "custom", "s", "palette");
-		g_menu_insert_item (priv->annot_menu, 0, item);
-
-		gtk_popover_menu_add_child (GTK_POPOVER_MENU (priv->view_popup),
-		                            priv->annot_menu_child, "palette");
-
-		priv->annot_menu_child_added = TRUE;
-	}
-
-	if (!(can_annotate && has_selection) && priv->annot_menu_child_added) {
-		g_menu_remove_all (priv->annot_menu);
-
-		priv->annot_menu_child_added = FALSE;
-	}
 }
 
 static void
@@ -4524,9 +4513,11 @@ pps_document_view_init (PpsDocumentView *pps_doc_view)
 	                                 pps_doc_view);
 	gtk_widget_insert_action_group (GTK_WIDGET (pps_doc_view),
 	                                "doc", G_ACTION_GROUP (priv->document_action_group));
-	// These are only enabled once the search has started
+	// These are only enabled once the user takes action
 	pps_document_view_set_action_enabled (pps_doc_view, "find-next", FALSE);
 	pps_document_view_set_action_enabled (pps_doc_view, "find-previous", FALSE);
+	pps_document_view_set_action_enabled (pps_doc_view, "add-highlight-annotation", FALSE);
+	pps_document_view_set_action_enabled (pps_doc_view, "copy", FALSE);
 
 	page_cache_mb = g_settings_get_uint (priv->settings,
 	                                     GS_PAGE_CACHE_SIZE);