TL;DR: User can make ssh-add go into infinite loop by setting environment variable to script that gives bad pw and rc=0.

Reproduction:

############ Setup:

eval $(ssh-agent -s)

umask 077
ssh-keygen -N asdfasdf -f /tmp/id_rsa

function testit() {
  ssh-add -D 2> /dev/null
  env DISPLAY=:0 SSH_ASKPASS="$1" ssh-add /tmp/id_rsa < /dev/null
}

function createit() {
  echo "echo '$1'; exit $2" > /tmp/a.sh
  chmod a+rx /tmp/a.sh
}

######### Tests:

# Correct operation, non-script with zero RC:
testit /bin/true

# Correct operation, non-script with non-zero RC:
testit /bin/false

# Script with correct password and zero RC:
createit asdfasdf 0; testit /tmp/a.sh    # Identity added: /tmp/id_rsa (/tmp/id_rsa)

# Script with correct password and non-zero RC:
createit asdfasdf 0; testit /tmp/a.sh    # Not added

# Script with incorrect password and non-zero RC:
createit wrong_pw 1; testit /tmp/a.sh    # Not added

# Script with incorrect password and zero RC:
createit wrong_pw 0; testit /tmp/a.sh    # (Hangs, hit ctrl-C to stop it.)

######## Debugging:
# Tracing it:
strace -s 1024 -fo/tmp/a env DISPLAY=:0 SSH_ASKPASS=/tmp/a.sh ssh-add /tmp/id_rsa < /dev/null
# (Ctrl-C after a few seconds)

grep -m 15 execve /tmp/a | tail

16950 execve("/usr/bin/ssh-add", ["ssh-add", "/tmp/id_rsa.pub"], [/* 86 vars */]) = 0
16951 execve("/tmp/a.sh", ["/tmp/a.sh", "Enter passphrase for /tmp/id_rsa.pub: "], [/* 86 vars */]) = -1 ENOEXEC (Exec format error)
16951 execve("/bin/sh", ["/bin/sh", "/tmp/a.sh", "Enter passphrase for /tmp/id_rsa.pub: "], [/* 86 vars */]) = 0
16952 execve("/tmp/a.sh", ["/tmp/a.sh", "Bad passphrase, try again for /tmp/id_rsa.pub: "], [/* 86 vars */]) = -1 ENOEXEC (Exec format error)
16952 execve("/bin/sh", ["/bin/sh", "/tmp/a.sh", "Bad passphrase, try again for /tmp/id_rsa.pub: "], [/* 86 vars */]) = 0
16953 execve("/tmp/a.sh", ["/tmp/a.sh", "Bad passphrase, try again for /tmp/id_rsa.pub: "], [/* 86 vars */]) = -1 ENOEXEC (Exec format error)
16953 execve("/bin/sh", ["/bin/sh", "/tmp/a.sh", "Bad passphrase, try again for /tmp/id_rsa.pub: "], [/* 86 vars */]) = 0
16954 execve("/tmp/a.sh", ["/tmp/a.sh", "Bad passphrase, try again for /tmp/id_rsa.pub: "], [/* 86 vars */]) = -1 ENOEXEC (Exec format error)
16954 execve("/bin/sh", ["/bin/sh", "/tmp/a.sh", "Bad passphrase, try again for /tmp/id_rsa.pub: "], [/* 86 vars */]) = 0
16955 execve("/tmp/a.sh", ["/tmp/a.sh", "Bad passphrase, try again for /tmp/id_rsa.pub: "], [/* 86 vars */]) = -1 ENOEXEC (Exec format error)

##---------------

Maybe make ssh-add abort after three tries?  Infinite retries is really only useful for password cracking.

Thanks!