Bugzilla bug #77199: Make NSS build under OS/2. Thanks to Javier

Pedemonte <pedemont@us.ibm.com> for the patch.
Modified Files:
Tag: NSS_3_2_BRANCH
	coreconf/OS2.mk coreconf/rules.mk coreconf/ruleset.mk
	nss/lib/freebl/Makefile nss/lib/pkcs12/p12.h nss/lib/ssl/ssl.h
	nss/lib/util/secport.h

7 files changed, 78 insertions, 42 deletions

diff --git a/security/coreconf/OS2.mk b/security/coreconf/OS2.mk
index 1a85c76db..5f914f63b 100644
--- a/security/coreconf/OS2.mk
+++ b/security/coreconf/OS2.mk
@@ -153,7 +153,7 @@ NSPR_INCLUDE_DIR =
 
 
 ifdef BUILD_OPT
-OPTIMIZER		= -O+ -Oi 
+OPTIMIZER		= -Oi -G5
 DEFINES 		+= -UDEBUG -U_DEBUG -DNDEBUG
 DLLFLAGS		= /DLL /O:$@ /INC:_dllentry /MAP:$(@:.dll=.map)
 EXEFLAGS    		= -PMTYPE:VIO -OUT:$@ -MAP:$(@:.exe=.map) -nologo -NOE
diff --git a/security/coreconf/rules.mk b/security/coreconf/rules.mk
index 8d27188ed..64d623e8a 100644
--- a/security/coreconf/rules.mk
+++ b/security/coreconf/rules.mk
@@ -279,6 +279,13 @@ alltags:
 	find . -name dist -prune -o \( -name '*.[hc]' -o -name '*.cp' -o -name '*.cpp' \) -print | xargs etags -a
 	find . -name dist -prune -o \( -name '*.[hc]' -o -name '*.cp' -o -name '*.cpp' \) -print | xargs ctags -a
 
+ifdef XP_OS2_VACPP
+# list of libs (such as -lnspr4) do not work for our compiler
+# change it to be $(DIST)/lib/nspr4.lib
+EXTRA_SHARED_LIBS := $(filter-out -L%,$(EXTRA_SHARED_LIBS))
+EXTRA_SHARED_LIBS := $(patsubst -l%,$(DIST)/lib/%.$(LIB_SUFFIX),$(EXTRA_SHARED_LIBS))
+endif
+
 $(PROGRAM): $(OBJS) $(EXTRA_LIBS)
 	@$(MAKE_OBJDIR)
 ifeq ($(OS_ARCH),WINNT)
@@ -325,7 +332,7 @@ $(IMPORT_LIBRARY): $(SHARED_LIBRARY)
 endif
 
 ifeq ($(OS_ARCH),OS2)
-$(IMPORT_LIBRARY): $(OBJS)
+$(IMPORT_LIBRARY): $(SHARED_LIBRARY)
 	rm -f $@
 	$(IMPLIB) $@ $(patsubst %.lib,%.dll.def,$@)
 	$(RANLIB) $@
@@ -371,7 +378,16 @@ ifeq ($(OS_ARCH),OS2)
 	@cmd /C "echo CODE    LOADONCALL MOVEABLE DISCARDABLE >>$@.def"
 	@cmd /C "echo DATA    PRELOAD MOVEABLE MULTIPLE NONSHARED >>$@.def"	
 	@cmd /C "echo EXPORTS >>$@.def"
-	@cmd /C "$(FILTER) $(OBJS) >>$@.def"
+	$(FILTER) $(OBJS) >>$@.def
+ifdef SUB_SHLOBJS
+	@echo Number of words in OBJ list = $(words $(SUB_SHLOBJS))
+	@echo If above number is over 100, need to reedit coreconf/rules.mk
+	-$(FILTER) $(wordlist 1,20,$(SUB_SHLOBJS)) >>$@.def
+	-$(FILTER) $(wordlist 21,40,$(SUB_SHLOBJS)) >>$@.def
+	-$(FILTER) $(wordlist 41,60,$(SUB_SHLOBJS)) >>$@.def
+	-$(FILTER) $(wordlist 61,80,$(SUB_SHLOBJS)) >>$@.def
+	-$(FILTER) $(wordlist 81,100,$(SUB_SHLOBJS)) >>$@.def
+endif
 endif #OS2
 ifdef XP_OS2_VACPP
 	$(MKSHLIB) $(DLLFLAGS) $(LDFLAGS) $(OBJS) $(SUB_SHLOBJS) $(LD_LIBS) $(EXTRA_LIBS) $(EXTRA_SHARED_LIBS) $@.def
diff --git a/security/coreconf/ruleset.mk b/security/coreconf/ruleset.mk
index 3733c41fd..6ddb78636 100644
--- a/security/coreconf/ruleset.mk
+++ b/security/coreconf/ruleset.mk
@@ -160,11 +160,17 @@ ifdef LIBRARY_NAME
 			IMPORT_LIBRARY = $(OBJDIR)/$(LIBRARY_NAME)$(LIBRARY_VERSION)32$(JDK_DEBUG_SUFFIX).lib
 		endif
 	else
-		LIBRARY = $(OBJDIR)/lib$(LIBRARY_NAME).$(LIB_SUFFIX)
-		ifeq ($(OS_ARCH)$(OS_RELEASE), AIX4.1)
-			SHARED_LIBRARY = $(OBJDIR)/lib$(LIBRARY_NAME)$(LIBRARY_VERSION)_shr$(JDK_DEBUG_SUFFIX).a
+		ifeq ($(OS_ARCH), OS2)
+			LIBRARY = $(OBJDIR)/lib$(LIBRARY_NAME).lib
+			SHARED_LIBRARY = $(OBJDIR)/$(LIBRARY_NAME)$(LIBRARY_VERSION)$(JDK_DEBUG_SUFFIX).dll
+			IMPORT_LIBRARY = $(OBJDIR)/$(LIBRARY_NAME)$(LIBRARY_VERSION)$(JDK_DEBUG_SUFFIX).lib
 		else
-			SHARED_LIBRARY = $(OBJDIR)/lib$(LIBRARY_NAME)$(LIBRARY_VERSION)$(JDK_DEBUG_SUFFIX).$(DLL_SUFFIX)
+			LIBRARY = $(OBJDIR)/lib$(LIBRARY_NAME).$(LIB_SUFFIX)
+			ifeq ($(OS_ARCH)$(OS_RELEASE), AIX4.1)
+				SHARED_LIBRARY = $(OBJDIR)/lib$(LIBRARY_NAME)$(LIBRARY_VERSION)_shr$(JDK_DEBUG_SUFFIX).a
+			else
+				SHARED_LIBRARY = $(OBJDIR)/lib$(LIBRARY_NAME)$(LIBRARY_VERSION)$(JDK_DEBUG_SUFFIX).$(DLL_SUFFIX)
+			endif
 		endif
 	endif
 endif
diff --git a/security/nss/lib/freebl/Makefile b/security/nss/lib/freebl/Makefile
index 78cee89a0..4fe9f4258 100644
--- a/security/nss/lib/freebl/Makefile
+++ b/security/nss/lib/freebl/Makefile
@@ -82,6 +82,9 @@ ifneq ($(OS_TARGET),WIN16)
 endif
 endif
 
+ifeq ($(OS_ARCH),OS2)
+CPORLN = cp
+endif
 
 ifeq ($(OS_ARCH),IRIX)
 ifeq ($(USE_N32),1)
diff --git a/security/nss/lib/pkcs12/p12.h b/security/nss/lib/pkcs12/p12.h
index eac067c93..14a8dd385 100644
--- a/security/nss/lib/pkcs12/p12.h
+++ b/security/nss/lib/pkcs12/p12.h
@@ -40,31 +40,39 @@
 #include "secpkcs7.h"
 #include "p12t.h"
 
-typedef int (* PKCS12OpenFunction)(void *arg);
-typedef int (* PKCS12ReadFunction)(void *arg, unsigned char *buffer, 
-				   unsigned int *lenRead, unsigned int maxLen);
-typedef int (* PKCS12WriteFunction)(void *arg, unsigned char *buffer, 
-				    unsigned int *bufLen, unsigned int *lenWritten);
-typedef int (* PKCS12CloseFunction)(void *arg);
-typedef SECStatus (* PKCS12UnicodeConvertFunction)(PRArenaPool *arena,
-						   SECItem *dest, SECItem *src,
-						   PRBool toUnicode, 
-						   PRBool swapBytes);
-typedef void (* SEC_PKCS12EncoderOutputCallback)(void *arg, const char *buf,
-						 unsigned long len);
-typedef void (* SEC_PKCS12DecoderOutputCallback)(void *arg, const char *buf,
-						 unsigned long len);
-typedef SECItem * (* SEC_PKCS12NicknameCollisionCallback)(SECItem *old_nickname,
-							  PRBool *cancel,
-							  void *arg);
-
-
-
-
-typedef SECStatus (*digestOpenFn)(void *arg, PRBool readData);
-typedef SECStatus (*digestCloseFn)(void *arg, PRBool removeFile);
-typedef int (*digestIOFn)(void *arg, unsigned char *buf, 
-			  unsigned long len);
+typedef int (PR_CALLBACK * PKCS12OpenFunction)(void *arg);
+typedef int (PR_CALLBACK * PKCS12ReadFunction)(void *arg,
+                                               unsigned char *buffer, 
+                                               unsigned int *lenRead,
+                                               unsigned int maxLen);
+typedef int (PR_CALLBACK * PKCS12WriteFunction)(void *arg,
+                                                unsigned char *buffer, 
+                                                unsigned int *bufLen,
+                                                unsigned int *lenWritten);
+typedef int (PR_CALLBACK * PKCS12CloseFunction)(void *arg);
+typedef SECStatus (PR_CALLBACK * PKCS12UnicodeConvertFunction)(
+                                 PRArenaPool *arena,
+                                 SECItem *dest, SECItem *src,
+                                 PRBool toUnicode,
+                                 PRBool swapBytes);
+typedef void (PR_CALLBACK * SEC_PKCS12EncoderOutputCallback)(
+                            void *arg, const char *buf,
+                            unsigned long len);
+typedef void (PR_CALLBACK * SEC_PKCS12DecoderOutputCallback)(
+                            void *arg, const char *buf,
+                            unsigned long len);
+typedef SECItem * (PR_CALLBACK * SEC_PKCS12NicknameCollisionCallback)(
+                                 SECItem *old_nickname,
+                                 PRBool *cancel,
+                                 void *arg);
+
+
+
+
+typedef SECStatus (PR_CALLBACK *digestOpenFn)(void *arg, PRBool readData);
+typedef SECStatus (PR_CALLBACK *digestCloseFn)(void *arg, PRBool removeFile);
+typedef int (PR_CALLBACK *digestIOFn)(void *arg, unsigned char *buf, 
+                                      unsigned long len);
 
 typedef struct SEC_PKCS12ExportContextStr SEC_PKCS12ExportContext;
 typedef struct SEC_PKCS12SafeInfoStr SEC_PKCS12SafeInfo;
diff --git a/security/nss/lib/ssl/ssl.h b/security/nss/lib/ssl/ssl.h
index d5cc34a84..7cc7616f5 100644
--- a/security/nss/lib/ssl/ssl.h
+++ b/security/nss/lib/ssl/ssl.h
@@ -203,8 +203,9 @@ SSL_IMPORT CERTCertificate *SSL_PeerCertificate(PRFileDesc *fd);
 ** (because of SSL_REQUIRE_CERTIFICATE in SSL_Enable) to authenticate the
 ** certificate.
 */
-typedef SECStatus (*SSLAuthCertificate)(void *arg, PRFileDesc *fd, 
-					PRBool checkSig, PRBool isServer);
+typedef SECStatus (PR_CALLBACK *SSLAuthCertificate)(void *arg, PRFileDesc *fd, 
+                                                    PRBool checkSig,
+                                                    PRBool isServer);
 
 SSL_IMPORT SECStatus SSL_AuthCertificateHook(PRFileDesc *fd, 
 					     SSLAuthCertificate f,
@@ -221,10 +222,11 @@ SSL_IMPORT SECStatus SSL_AuthCertificate(void *arg, PRFileDesc *fd,
  *	pRetCert - pointer to pointer to cert, for return of cert
  *	pRetKey - pointer to key pointer, for return of key
  */
-typedef SECStatus (*SSLGetClientAuthData)(void *arg, PRFileDesc *fd,
-				    CERTDistNames *caNames,
-				    CERTCertificate **pRetCert,/*return */
-				    SECKEYPrivateKey **pRetKey);/* return */
+typedef SECStatus (PR_CALLBACK *SSLGetClientAuthData)(void *arg,
+                                PRFileDesc *fd,
+                                CERTDistNames *caNames,
+                                CERTCertificate **pRetCert,/*return */
+                                SECKEYPrivateKey **pRetKey);/* return */
 
 /*
  * Set the client side callback for SSL to retrieve user's private key
@@ -249,7 +251,7 @@ SSL_IMPORT SECStatus SSL_SetPKCS11PinArg(PRFileDesc *fd, void *a);
 ** by the client.  The client app can decide that it actually likes the
 ** cert by some external means and restart the connection.
 */
-typedef SECStatus (*SSLBadCertHandler)(void *arg, PRFileDesc *fd);
+typedef SECStatus (PR_CALLBACK *SSLBadCertHandler)(void *arg, PRFileDesc *fd);
 SSL_IMPORT SECStatus SSL_BadCertHook(PRFileDesc *fd, SSLBadCertHandler f, 
 				     void *arg);
 
@@ -313,7 +315,8 @@ SSL_IMPORT SECStatus SSL_InheritMPServerSIDCache(const char * envString);
 ** Set the callback on a particular socket that gets called when we finish
 ** performing a handshake.
 */
-typedef void (*SSLHandshakeCallback)(PRFileDesc *fd, void *client_data);
+typedef void (PR_CALLBACK *SSLHandshakeCallback)(PRFileDesc *fd,
+                                                 void *client_data);
 SSL_IMPORT SECStatus SSL_HandshakeCallback(PRFileDesc *fd, 
 			          SSLHandshakeCallback cb, void *client_data);
 
diff --git a/security/nss/lib/util/secport.h b/security/nss/lib/util/secport.h
index 8ccba60ec..3c92676a2 100644
--- a/security/nss/lib/util/secport.h
+++ b/security/nss/lib/util/secport.h
@@ -217,12 +217,12 @@ char *PORT_Strdup(const char *);
 
 #endif /* XP_STRING_FUNCS */
 
-typedef PRBool (* PORTCharConversionWSwapFunc) (PRBool toUnicode,
+typedef PRBool (PR_CALLBACK * PORTCharConversionWSwapFunc) (PRBool toUnicode,
 			unsigned char *inBuf, unsigned int inBufLen,
 			unsigned char *outBuf, unsigned int maxOutBufLen,
 			unsigned int *outBufLen, PRBool swapBytes);
 
-typedef PRBool (* PORTCharConversionFunc) (PRBool toUnicode,
+typedef PRBool (PR_CALLBACK * PORTCharConversionFunc) (PRBool toUnicode,
 			unsigned char *inBuf, unsigned int inBufLen,
 			unsigned char *outBuf, unsigned int maxOutBufLen,
 			unsigned int *outBufLen);