From 5995a48d7b8b9a73a664bf03ef726c7161492da0 Mon Sep 17 00:00:00 2001
From: Alex Plate <aleksei.plate@jetbrains.com>
Date: Tue, 17 Dec 2024 20:36:51 +0200
Subject: [PATCH] [VIM-3577] Replace weak reference to focused editor to needed
 information

The `editorInFocus` used the weak reference in order to avoid editor leaks. However, if the user is unlucky, the GC may be called in between the console closing and switching focus to the new editor. In this case, the logic breaks and the Editor remains in the insert mode.
Now, we store only the required information about the last used editor.
---
 .../idea/vim/listener/IJEditorFocusListener.kt     | 14 ++++++++------
 .../kotlin/com/maddyhome/idea/vim/KeyHandler.kt    | 13 ++++++-------
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/src/main/java/com/maddyhome/idea/vim/listener/IJEditorFocusListener.kt b/src/main/java/com/maddyhome/idea/vim/listener/IJEditorFocusListener.kt
index 8f002bf4bb..a3616b4285 100644
--- a/src/main/java/com/maddyhome/idea/vim/listener/IJEditorFocusListener.kt
+++ b/src/main/java/com/maddyhome/idea/vim/listener/IJEditorFocusListener.kt
@@ -13,6 +13,7 @@ import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.editor.Editor
 import com.intellij.openapi.editor.EditorKind
 import com.maddyhome.idea.vim.KeyHandler
+import com.maddyhome.idea.vim.LastUsedEditorInfo
 import com.maddyhome.idea.vim.VimPlugin
 import com.maddyhome.idea.vim.api.ExecutionContext
 import com.maddyhome.idea.vim.api.VimEditor
@@ -31,10 +32,9 @@ import com.maddyhome.idea.vim.state.mode.Mode
  */
 class IJEditorFocusListener : EditorListener {
   override fun focusGained(editor: VimEditor) {
-    val oldEditor = KeyHandler.getInstance().editorInFocus
-    if (oldEditor != null && oldEditor.ij == editor.ij) return
-
-    KeyHandler.getInstance().editorInFocus = editor
+    val oldEditorInfo = KeyHandler.getInstance().lastUsedEditorInfo
+    val currentEditorHashCode = editor.ij.hashCode()
+    if (oldEditorInfo.hash == currentEditorHashCode) return
 
     // We add Vim bindings to all opened editors, including editors used as UI controls rather than just project file
     // editors. This includes editors used as part of the UI, such as the VCS commit message, or used as read-only
@@ -57,15 +57,17 @@ class IJEditorFocusListener : EditorListener {
     // to know that a read-only editor that is hosting a console view with a running process can be treated as writable.
 
     val ijEditor = editor.ij
+    val isCurrentEditorTerminal = isTerminal(ijEditor)
 
+    KeyHandler.getInstance().lastUsedEditorInfo = LastUsedEditorInfo(currentEditorHashCode, isCurrentEditorTerminal)
 
     val switchToInsertMode = Runnable {
       val context: ExecutionContext = injector.executionContextManager.getEditorExecutionContext(editor)
       VimPlugin.getChange().insertBeforeCursor(editor, context)
     }
-    if (isTerminal(ijEditor) && !ijEditor.inInsertMode) {
+    if (isCurrentEditorTerminal && !ijEditor.inInsertMode) {
       switchToInsertMode.run()
-    } else if (ijEditor.isInsertMode && ((oldEditor != null && isTerminal(oldEditor.ij)) || !ijEditor.document.isWritable)) {
+    } else if (ijEditor.isInsertMode && (oldEditorInfo.isTerminal || !ijEditor.document.isWritable)) {
       val context: ExecutionContext = injector.executionContextManager.getEditorExecutionContext(editor)
       val mode = injector.vimState.mode
       when (mode) {
diff --git a/vim-engine/src/main/kotlin/com/maddyhome/idea/vim/KeyHandler.kt b/vim-engine/src/main/kotlin/com/maddyhome/idea/vim/KeyHandler.kt
index 6d058466be..e9b2306189 100644
--- a/vim-engine/src/main/kotlin/com/maddyhome/idea/vim/KeyHandler.kt
+++ b/vim-engine/src/main/kotlin/com/maddyhome/idea/vim/KeyHandler.kt
@@ -36,7 +36,6 @@ import com.maddyhome.idea.vim.state.VimStateMachine
 import com.maddyhome.idea.vim.state.mode.Mode
 import com.maddyhome.idea.vim.state.mode.ReturnTo
 import com.maddyhome.idea.vim.state.mode.returnTo
-import java.lang.ref.WeakReference
 import javax.swing.KeyStroke
 
 /**
@@ -47,7 +46,6 @@ import javax.swing.KeyStroke
 // 1. avoid using handleKeyRecursionCount & shouldRecord
 // 2. maybe we can live without allowKeyMappings: Boolean & mappingCompleted: Boolean
 class KeyHandler {
-  private var editorInFocusReference: WeakReference<VimEditor>? = null
   private val keyConsumers: List<KeyConsumer> = listOf(ModalInputConsumer(), MappingProcessor, CommandCountConsumer(), DeleteCommandConsumer(), EditorResetConsumer(), CharArgumentConsumer(), RegisterConsumer(), DigraphConsumer(), CommandConsumer(), SelectRegisterConsumer(), ModeInputConsumer())
   private var handleKeyRecursionCount = 0
 
@@ -64,11 +62,7 @@ class KeyHandler {
   val keyStack: KeyStack = KeyStack()
   val modalEntryKeys: MutableList<KeyStroke> = ArrayList()
 
-  var editorInFocus: VimEditor?
-    get() = editorInFocusReference?.get()
-    set(value) {
-      editorInFocusReference = WeakReference(value)
-    }
+  var lastUsedEditorInfo: LastUsedEditorInfo = LastUsedEditorInfo(-1, false)
 
   /**
    * This is the main key handler for the Vim plugin. Every keystroke not handled directly by Idea is sent here for
@@ -482,3 +476,8 @@ sealed interface KeyProcessResult {
 }
 
 typealias KeyProcessing = (KeyHandlerState, VimEditor, ExecutionContext) -> Unit
+
+data class LastUsedEditorInfo(
+  val hash: Int,
+  val isTerminal: Boolean,
+)
\ No newline at end of file