diff --git a/plugins/ssh-agent/ssh-agent.plugin.zsh b/plugins/ssh-agent/ssh-agent.plugin.zsh
index 20f97c6f..794dabe1 100644
--- a/plugins/ssh-agent/ssh-agent.plugin.zsh
+++ b/plugins/ssh-agent/ssh-agent.plugin.zsh
@@ -1,3 +1,8 @@
+if [[ ! -d $HOME/.ssh ]]; then
+    echo $HOME/.ssh not found, exiting...
+    return 1
+fi
+
 typeset _agent_forwarding _ssh_env_cache
 
 function _start_agent() {