diff --git a/plugins/history-substring-search/history-substring-search.plugin.zsh b/plugins/history-substring-search/history-substring-search.plugin.zsh
index 25fd3a2d..7883a65f 100644
--- a/plugins/history-substring-search/history-substring-search.plugin.zsh
+++ b/plugins/history-substring-search/history-substring-search.plugin.zsh
@@ -13,11 +13,14 @@ fi
 
 
 # Bind terminal-specific up and down keys
-
+# Bind in both emacs and vi modes so it works in both, and is not
+# sensitive to whether this is loaded before or after the vi-mode plugin
 if [[ -n "$terminfo[kcuu1]" ]]; then
-  bindkey "$terminfo[kcuu1]" history-substring-search-up
+  bindkey -M emacs "$terminfo[kcuu1]" history-substring-search-up
+  bindkey -M viins "$terminfo[kcuu1]" history-substring-search-up
 fi
 if [[ -n "$terminfo[kcud1]" ]]; then
-  bindkey "$terminfo[kcud1]" history-substring-search-down
+  bindkey -M emacs "$terminfo[kcud1]" history-substring-search-down
+  bindkey -M viins "$terminfo[kcud1]" history-substring-search-down
 fi
 
diff --git a/plugins/history-substring-search/update-from-upstream.zsh b/plugins/history-substring-search/update-from-upstream.zsh
index 6e6cca5d..81e1942a 100755
--- a/plugins/history-substring-search/update-from-upstream.zsh
+++ b/plugins/history-substring-search/update-from-upstream.zsh
@@ -76,10 +76,12 @@ cat >> $plugin_basename.plugin.zsh <<EOF
 # Bind terminal-specific up and down keys
 
 if [[ -n "\$terminfo[kcuu1]" ]]; then
-  bindkey "\$terminfo[kcuu1]" history-substring-search-up
+  bindkey -M emacs "\$terminfo[kcuu1]" history-substring-search-up
+  bindkey -M viins "\$terminfo[kcuu1]" history-substring-search-up
 fi
 if [[ -n "\$terminfo[kcud1]" ]]; then
-  bindkey "\$terminfo[kcud1]" history-substring-search-down
+  bindkey -M emacs "\$terminfo[kcud1]" history-substring-search-down
+  bindkey -M viins "\$terminfo[kcud1]" history-substring-search-down
 fi
 
 EOF