If /etc/nologin is given, the loginrestrictions() function in auth.c will
decline any non-root login. So the users will get some kind of wrong
password message and do not see any content of /etc/nologin at all.
The below fix will correct this behavior. Unfortunately it triggers
another - possible AIX 4.3.3 only - bug which will cause the server
process to hang forever.
I have a somewhat ugly workaround and also the systemcall wich causes
the server to hang. May be there is another way to fix this, but i don't
know much enough about the internals of the openssh sourcecode.

After inserting some extra debug statements i got the following
debug-trace when the server will hang:

# sshd -ddd
[...]
debug3: tty_parse_modes: 90 1
debug3: tty_parse_modes: 91 1
debug3: tty_parse_modes: 92 0
debug3: tty_parse_modes: 93 0
debug1: exit session_input_channel_req
debug1: server_input_channel_req: channel 0 request x11-req reply 0
debug1: session_by_channel: session 0 channel 0
debug1: session_input_channel_req: session 0 req x11-req
debug1: fd 10 setting O_NONBLOCK
debug2: fd 10 is O_NONBLOCK
debug1: channel 1: new [X11 inet listener]
debug1: exit session_input_channel_req
debug1: server_input_channel_req: channel 0 request shell reply 0
debug1: session_by_channel: session 0 channel 0
debug1: session_input_channel_req: session 0 req shell
debug1: calling session_shell_req
debug1: calling packet_check_eom
debug1: calling do_exec
debug1: calling do_exec_pty
setsid: Not owner
debug1: Received SIGCHLD.
debug1: parent+
debug1: parent++
^rz192:~
#

The "parent+" debug statements are in patch No 3.

Index: auth.c
===================================================================
RCS file: /usr/local/.cvs/auth/openssh/auth.c,v
retrieving revision 1.1.1.5
diff -c -r1.1.1.5 auth.c
*** auth.c	2002/03/08 07:55:38	1.1.1.5
--- auth.c	2002/03/15 12:54:01
***************
*** 177,183 ****
  	}
  
  #ifdef WITH_AIXAUTHENTICATE
! 	if (loginrestrictions(pw->pw_name, S_RLOGIN, NULL, &loginmsg) != 0) {
  		if (loginmsg && *loginmsg) {
  			/* Remove embedded newlines (if any) */
  			char *p;
--- 177,190 ----
  	}
  
  #ifdef WITH_AIXAUTHENTICATE
! 	/*
! 	 * To let the user read a possible message in /etc/nologin we have to
! 	 * accept him here. Otherwise the message will just be logged and the
! 	 * user gets only a permission denied message. 
! 	 */
! 	if (loginrestrictions(pw->pw_name, S_RLOGIN, NULL, &loginmsg) != 0 &&
! 	    (errno != EPERM ||
!              stat(_PATH_NOLOGIN, &st) != 0)) {
  		if (loginmsg && *loginmsg) {
  			/* Remove embedded newlines (if any) */
  			char *p;

Index: session.c
===================================================================
RCS file: /usr/local/.cvs/auth/openssh/session.c,v
retrieving revision 1.1.1.7
diff -c -r1.1.1.7 session.c
*** session.c	2002/03/08 07:55:51	1.1.1.7
--- session.c	2002/03/20 07:59:02
***************
*** 1094,1099 ****
--- 1094,1107 ----
  		while (fgets(buf, sizeof(buf), f))
  			fputs(buf, stderr);
  		fclose(f);
+ #ifdef _AIX
+ 		/*
+ 		 * This is ugly, but otherwise AIX (at least 4.3.3ML8)
+ 		 * will get stuck forever in the close(ttyfd) call of
+ 		 * the parent process in do_exec_pty().
+ 		 */
+ 		sleep(2);
+ #endif /* _AIX */
  		exit(254);
  	}
  }

Index: session.c
===================================================================
RCS file: /usr/local/.cvs/auth/openssh/session.c,v
retrieving revision 1.1.1.7
diff -c -r1.1.1.7 session.c
*** session.c	2002/03/08 07:55:51	1.1.1.7
--- session.c	2002/03/19 15:17:52
***************
*** 572,590 ****
--- 572,594 ----
  	if (is_winnt)
  		cygwin_set_impersonation_token(INVALID_HANDLE_VALUE);
  #endif
+ debug("parent+");
  	if (pid < 0)
  		packet_disconnect("fork failed: %.100s", strerror(errno));
  	s->pid = pid;
  
+ debug("parent++");
  	/* Parent.  Close the slave side of the pseudo tty. */
  	close(ttyfd);
  
+ debug("parent+++");
  	/*
  	 * Create another descriptor of the pty master side for use as the
  	 * standard input.  We could use the original descriptor, but this
  	 * simplifies code in server_loop.  The descriptor is bidirectional.
  	 */
  	fdout = dup(ptyfd);
+ debug("parent++++");
  	if (fdout < 0)
  		packet_disconnect("dup #1 failed: %.100s", strerror(errno));