Merge mikael-ronstr-ms-dator.local:/Users/mikron/mysql_clones/mysql-5.0-ndb

into  mikael-ronstr-ms-dator.local:/Users/mikron/mysql_clones/mysql-5.1-ndb


BUILD/Makefile.am:
  Auto merged
config/ac-macros/ha_ndbcluster.m4:
  Auto merged
storage/ndb/src/common/transporter/SCI_Transporter.cpp:
  Auto merged
storage/ndb/src/common/transporter/SCI_Transporter.hpp:
  Auto merged
libmysqld/Makefile.am:
  Manual merge
libmysqld/examples/Makefile.am:
  Manual merge

8 files changed, 73 insertions, 186 deletions

diff --git a/BUILD/Makefile.am b/BUILD/Makefile.am
index 46814a077de..82e31c4a47b 100644
--- a/BUILD/Makefile.am
+++ b/BUILD/Makefile.am
@@ -28,6 +28,7 @@ EXTRA_DIST =		FINISH.sh \
 			compile-alpha-debug \
 			compile-amd64-debug-max \
 			compile-amd64-max \
+			compile-amd64-max-sci \
 			compile-darwin-mwcc \
 			compile-dist \
 			compile-hpux11-parisc2-aCC \
@@ -52,6 +53,7 @@ EXTRA_DIST =		FINISH.sh \
 			compile-pentium-valgrind-max \
 			compile-pentium64-debug \
 			compile-pentium64-debug-max \
+			compile-pentium64-max-sci \
 			compile-pentium64-valgrind-max \
 			compile-ppc \
 			compile-ppc-debug \
diff --git a/BUILD/compile-amd64-max-sci b/BUILD/compile-amd64-max-sci
new file mode 100644
index 00000000000..4afa9004742
--- /dev/null
+++ b/BUILD/compile-amd64-max-sci
@@ -0,0 +1,8 @@
+#! /bin/sh
+
+path=`dirname $0`
+. "$path/SETUP.sh"
+extra_flags="$amd64_cflags $fast_cflags -g"
+extra_configs="$amd64_configs $max_configs --with-ndb-sci=/opt/DIS"
+
+. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium64-max-sci b/BUILD/compile-pentium64-max-sci
new file mode 100644
index 00000000000..9ebb1988475
--- /dev/null
+++ b/BUILD/compile-pentium64-max-sci
@@ -0,0 +1,9 @@
+#! /bin/sh
+
+path=`dirname $0`
+. "$path/SETUP.sh"
+
+extra_flags="$pentium64_cflags $fast_cflags -g"
+extra_configs="$pentium_configs $max_configs --with-ndb-sci=/opt/DIS"
+
+. "$path/FINISH.sh"
diff --git a/config/ac-macros/ha_ndbcluster.m4 b/config/ac-macros/ha_ndbcluster.m4
index 2a54638b3ae..b9ba68d51c0 100644
--- a/config/ac-macros/ha_ndbcluster.m4
+++ b/config/ac-macros/ha_ndbcluster.m4
@@ -43,7 +43,7 @@ AC_DEFUN([MYSQL_CHECK_NDB_OPTIONS], [
       if test -f "$mysql_sci_dir/lib/libsisci.a" -a \ 
               -f "$mysql_sci_dir/include/sisci_api.h"; then
         NDB_SCI_INCLUDES="-I$mysql_sci_dir/include"
-        NDB_SCI_LIBS="-L$mysql_sci_dir/lib -lsisci"
+        NDB_SCI_LIBS="$mysql_sci_dir/lib/libsisci.a"
         AC_MSG_RESULT([-- including sci transporter])
         AC_DEFINE([NDB_SCI_TRANSPORTER], [1],
                   [Including Ndb Cluster DB sci transporter])
diff --git a/libmysqld/Makefile.am b/libmysqld/Makefile.am
index cf5971affe4..024b85eed8e 100644
--- a/libmysqld/Makefile.am
+++ b/libmysqld/Makefile.am
@@ -95,6 +95,7 @@ INC_LIB=	$(top_builddir)/regex/libregex.a \
 		$(top_builddir)/strings/libmystrings.a \
 		$(top_builddir)/dbug/libdbug.a \
 		$(top_builddir)/vio/libvio.a \
+                @NDB_SCI_LIBS@ \
 		@mysql_plugin_libs@ \
 		$(yassl_inc_libs)
 
diff --git a/libmysqld/examples/Makefile.am b/libmysqld/examples/Makefile.am
index 3e4769c564d..ec5bbbb86e5 100644
--- a/libmysqld/examples/Makefile.am
+++ b/libmysqld/examples/Makefile.am
@@ -36,7 +36,8 @@ INCLUDES =	-I$(top_builddir)/include -I$(top_srcdir)/include -I$(srcdir) \
 		-I$(top_srcdir) -I$(top_srcdir)/client -I$(top_srcdir)/regex \
 		$(openssl_includes)
 LIBS =		@LIBS@ @WRAPLIBS@ @CLIENT_LIBS@ $(yassl_libs)
-LDADD =		@CLIENT_EXTRA_LDFLAGS@ ../libmysqld.a @LIBDL@ $(CXXLDFLAGS)
+LDADD =		@CLIENT_EXTRA_LDFLAGS@ ../libmysqld.a @LIBDL@ $(CXXLDFLAGS) \
+                @NDB_SCI_LIBS@
 
 mysqltest_embedded_LINK = $(CXXLINK)
 mysqltest_embedded_SOURCES =	mysqltest.c
diff --git a/storage/ndb/src/common/transporter/SCI_Transporter.cpp b/storage/ndb/src/common/transporter/SCI_Transporter.cpp
index 138b79acb51..0720fe84973 100644
--- a/storage/ndb/src/common/transporter/SCI_Transporter.cpp
+++ b/storage/ndb/src/common/transporter/SCI_Transporter.cpp
@@ -65,13 +65,10 @@ SCI_Transporter::SCI_Transporter(TransporterRegistry &t_reg,
  
    
   m_initLocal=false; 
-  m_swapCounter=0; 
   m_failCounter=0; 
   m_remoteNodes[0]=remoteSciNodeId0; 
   m_remoteNodes[1]=remoteSciNodeId1; 
   m_adapters = nAdapters;   
-  // The maximum number of times to try and create,  
-  // start and destroy a sequence 
   m_ActiveAdapterId=0; 
   m_StandbyAdapterId=1; 
   
@@ -102,8 +99,6 @@ SCI_Transporter::SCI_Transporter(TransporterRegistry &t_reg,
   DBUG_VOID_RETURN;
 } 
  
- 
- 
 void SCI_Transporter::disconnectImpl() 
 { 
   DBUG_ENTER("SCI_Transporter::disconnectImpl");
@@ -129,7 +124,8 @@ void SCI_Transporter::disconnectImpl()
       
       if(err != SCI_ERR_OK)  { 
 	report_error(TE_SCI_UNABLE_TO_CLOSE_CHANNEL); 
-        DBUG_PRINT("error", ("Cannot close channel to the driver. Error code 0x%x",  
+        DBUG_PRINT("error",
+        ("Cannot close channel to the driver. Error code 0x%x",  
 		    err)); 
       } 
     } 
@@ -164,19 +160,18 @@ bool SCI_Transporter::initTransporter() {
   m_sendBuffer.m_buffer = new Uint32[m_sendBuffer.m_sendBufferSize / 4];
   m_sendBuffer.m_dataSize = 0;
  
-  DBUG_PRINT("info", ("Created SCI Send Buffer with buffer size %d and packet size %d",
+  DBUG_PRINT("info",
+  ("Created SCI Send Buffer with buffer size %d and packet size %d",
               m_sendBuffer.m_sendBufferSize, m_PacketSize * 4));
   if(!getLinkStatus(m_ActiveAdapterId) ||  
      (m_adapters > 1 &&
      !getLinkStatus(m_StandbyAdapterId))) { 
-    DBUG_PRINT("error", ("The link is not fully operational. Check the cables and the switches")); 
-    //reportDisconnect(remoteNodeId, 0); 
-    //doDisconnect(); 
+    DBUG_PRINT("error",
+    ("The link is not fully operational. Check the cables and the switches")); 
     //NDB should terminate 
     report_error(TE_SCI_LINK_ERROR); 
     DBUG_RETURN(false); 
   } 
-  
   DBUG_RETURN(true); 
 } // initTransporter()  
 
@@ -235,7 +230,8 @@ sci_error_t SCI_Transporter::initLocalSegment() {
       DBUG_PRINT("info", ("SCInode iD %d  adapter %d\n",  
 	         sciAdapters[i].localSciNodeId, i)); 
       if(err != SCI_ERR_OK) { 
-        DBUG_PRINT("error", ("Cannot open an SCI virtual device. Error code 0x%x", 
+        DBUG_PRINT("error",
+        ("Cannot open an SCI virtual device. Error code 0x%x", 
 		   err)); 
 	DBUG_RETURN(err); 
       } 
@@ -269,7 +265,8 @@ sci_error_t SCI_Transporter::initLocalSegment() {
 		      &err); 
      
     if(err != SCI_ERR_OK) { 
-      DBUG_PRINT("error", ("Local Segment is not accessible by an SCI adapter. Error code 0x%x\n",
+      DBUG_PRINT("error",
+    ("Local Segment is not accessible by an SCI adapter. Error code 0x%x\n",
                   err)); 
       DBUG_RETURN(err); 
     } 
@@ -303,15 +300,13 @@ sci_error_t SCI_Transporter::initLocalSegment() {
 			   &err); 
      
     if(err != SCI_ERR_OK) { 
-      DBUG_PRINT("error", ("Local Segment is not available for remote connections. Error code 0x%x\n",
+      DBUG_PRINT("error",
+   ("Local Segment is not available for remote connections. Error code 0x%x\n",
                  err)); 
       DBUG_RETURN(err); 
     } 
   } 
-  
-  
   setupLocalSegment(); 
-  
   DBUG_RETURN(err); 
    
 } // initLocalSegment() 
@@ -343,12 +338,6 @@ bool SCI_Transporter::doSend() {
     if(sizeToSend==4097) 
       i4097++; 
 #endif
-    if(startSequence(m_ActiveAdapterId)!=SCI_ERR_OK) { 
-      DBUG_PRINT("error", ("Start sequence failed")); 
-      report_error(TE_SCI_UNABLE_TO_START_SEQUENCE); 
-      return false; 
-    } 
-    
       
   tryagain:
     retry++;
@@ -374,119 +363,36 @@ bool SCI_Transporter::doSend() {
 		SCI_FLAG_ERROR_CHECK, 
 		&err);   
       
-      
       if (err != SCI_ERR_OK) { 
-      if(err == SCI_ERR_OUT_OF_RANGE) { 
-        DBUG_PRINT("error", ("Data transfer : out of range error")); 
-	goto tryagain; 
-      } 
-      if(err == SCI_ERR_SIZE_ALIGNMENT) { 
-        DBUG_PRINT("error", ("Data transfer : alignment error")); 
-        DBUG_PRINT("info", ("sendPtr 0x%x, sizeToSend = %d", sendPtr, sizeToSend));
-	goto tryagain; 
-      } 
-      if(err == SCI_ERR_OFFSET_ALIGNMENT) { 
-        DBUG_PRINT("error", ("Data transfer : offset alignment")); 
-	goto tryagain; 
-      }   
-      if(err == SCI_ERR_TRANSFER_FAILED) { 
-	//(m_TargetSegm[m_StandbyAdapterId].writer)->heavyLock(); 
-	if(getLinkStatus(m_ActiveAdapterId)) { 
-	  goto tryagain; 
-	}
-        if (m_adapters == 1) {
-          DBUG_PRINT("error", ("SCI Transfer failed"));
+        if (err == SCI_ERR_OUT_OF_RANGE ||
+            err == SCI_ERR_SIZE_ALIGNMENT ||
+            err == SCI_ERR_OFFSET_ALIGNMENT) { 
+          DBUG_PRINT("error", ("Data transfer error = %d", err));
           report_error(TE_SCI_UNRECOVERABLE_DATA_TFX_ERROR);
 	  return false; 
-        }
-	m_failCounter++; 
-	Uint32 temp=m_ActiveAdapterId;	    	     
-	switch(m_swapCounter) { 
-	case 0:  
-	  /**swap from active (0) to standby (1)*/ 
-	  if(getLinkStatus(m_StandbyAdapterId)) { 
-            DBUG_PRINT("error", ("Swapping from adapter 0 to 1")); 
+        } 
+        if(err == SCI_ERR_TRANSFER_FAILED) { 
+	  if(getLinkStatus(m_ActiveAdapterId))
+	    goto tryagain; 
+          if (m_adapters == 1) {
+            DBUG_PRINT("error", ("SCI Transfer failed"));
+            report_error(TE_SCI_UNRECOVERABLE_DATA_TFX_ERROR);
+	    return false; 
+          }
+	  m_failCounter++; 
+	  Uint32 temp=m_ActiveAdapterId;	    	     
+	  if (getLinkStatus(m_StandbyAdapterId)) { 
 	    failoverShmWriter();		 
 	    SCIStoreBarrier(m_TargetSegm[m_StandbyAdapterId].sequence,0); 
 	    m_ActiveAdapterId=m_StandbyAdapterId; 
 	    m_StandbyAdapterId=temp; 
-	    SCIRemoveSequence((m_TargetSegm[m_StandbyAdapterId].sequence),
-			      FLAGS,  
-			      &err); 
-	    if(err!=SCI_ERR_OK) { 
-	      report_error(TE_SCI_UNABLE_TO_REMOVE_SEQUENCE); 
-              DBUG_PRINT("error", ("Unable to remove sequence"));
-	      return false; 
-	    } 
-	    if(startSequence(m_ActiveAdapterId)!=SCI_ERR_OK) { 
-              DBUG_PRINT("error", ("Start sequence failed")); 
-	      report_error(TE_SCI_UNABLE_TO_START_SEQUENCE); 
-	      return false; 
-	    } 
-	    m_swapCounter++; 
-            DBUG_PRINT("info", ("failover complete")); 
-	    goto tryagain; 
-	  }  else {
-	    report_error(TE_SCI_UNRECOVERABLE_DATA_TFX_ERROR);
-            DBUG_PRINT("error", ("SCI Transfer failed")); 
-	    return false;
-	  }
-	  return false; 
-	  break; 
-	case 1: 
-	  /** swap back from 1 to 0 
-	      must check that the link is up */ 
-	  
-	  if(getLinkStatus(m_StandbyAdapterId)) { 
-	    failoverShmWriter(); 
-	    m_ActiveAdapterId=m_StandbyAdapterId; 
-	    m_StandbyAdapterId=temp; 
-            DBUG_PRINT("info", ("Swapping from 1 to 0"));	 
-	    if(createSequence(m_ActiveAdapterId)!=SCI_ERR_OK) { 
-              DBUG_PRINT("error", ("Unable to create sequence"));
-	      report_error(TE_SCI_UNABLE_TO_CREATE_SEQUENCE); 
-	      return false; 
-	    } 
-	    if(startSequence(m_ActiveAdapterId)!=SCI_ERR_OK) { 
-              DBUG_PRINT("error", ("startSequence failed... disconnecting")); 
-	      report_error(TE_SCI_UNABLE_TO_START_SEQUENCE); 
-	      return false; 
-	    } 
-	    
-	    SCIRemoveSequence((m_TargetSegm[m_StandbyAdapterId].sequence) 
-			      , FLAGS,  
-			      &err); 
-	    if(err!=SCI_ERR_OK) { 
-              DBUG_PRINT("error", ("Unable to remove sequence"));
-	      report_error(TE_SCI_UNABLE_TO_REMOVE_SEQUENCE); 
-	      return false;
-	    } 
-	    
-	    if(createSequence(m_StandbyAdapterId)!=SCI_ERR_OK) { 
-              DBUG_PRINT("error", ("Unable to create sequence on standby"));
-	      report_error(TE_SCI_UNABLE_TO_CREATE_SEQUENCE); 
-	      return false; 
-	    } 
-	    
-	    m_swapCounter=0; 
-	    
-            DBUG_PRINT("info", ("failover complete..")); 
-	    goto tryagain; 
-	    
+            DBUG_PRINT("error", ("Swapping from adapter %u to %u",
+                       m_StandbyAdapterId, m_ActiveAdapterId));
 	  } else {
-            DBUG_PRINT("error", ("Unrecoverable data transfer error")); 
 	    report_error(TE_SCI_UNRECOVERABLE_DATA_TFX_ERROR);
-	    return false;
+            DBUG_PRINT("error", ("SCI Transfer failed")); 
 	  }
-	  
-	  break; 
-	default: 
-          DBUG_PRINT("error", ("Unrecoverable data transfer error")); 
-	  report_error(TE_SCI_UNRECOVERABLE_DATA_TFX_ERROR); 
-	  return false; 
-	  break; 
-	}  
-      }
+        }
       } else { 
 	SHM_Writer * writer = (m_TargetSegm[m_ActiveAdapterId].writer);
 	writer->updateWritePtr(sizeToSend); 
@@ -497,7 +403,6 @@ bool SCI_Transporter::doSend() {
 	m_sendBuffer.m_dataSize = 0;
 	m_sendBuffer.m_forceSendLimit = sendLimit;
       } 
-      
     } else { 
       /** 
        * If we end up here, the SCI segment is full.  
@@ -552,15 +457,12 @@ void SCI_Transporter::setupLocalSegment()
    DBUG_VOID_RETURN;
 } //setupLocalSegment 
  
- 
- 
 void SCI_Transporter::setupRemoteSegment()   
 { 
    DBUG_ENTER("SCI_Transporter::setupRemoteSegment");
    Uint32 sharedSize = 0; 
    sharedSize =4096;   //start of the buffer is page aligned 
  
- 
    Uint32 sizeOfBuffer = m_BufferSize; 
    const Uint32 slack = MAX_MESSAGE_SIZE;
    sizeOfBuffer -= sharedSize; 
@@ -666,7 +568,6 @@ SCI_Transporter::init_remote()
         DBUG_PRINT("error", ("Error connecting segment, err 0x%x", err));
         DBUG_RETURN(false);
       }
-
     }
     // Map the remote memory segment into program space  
     for(Uint32 i=0; i < m_adapters ; i++) {
@@ -679,13 +580,14 @@ SCI_Transporter::init_remote()
                             FLAGS,
                             &err);
 
-
-        if(err!= SCI_ERR_OK) {
-          DBUG_PRINT("error", ("Cannot map a segment to the remote node %d. Error code 0x%x",m_RemoteSciNodeId, err));
-          //NDB SHOULD TERMINATE AND COMPUTER REBOOTED! 
-          report_error(TE_SCI_CANNOT_MAP_REMOTESEGMENT);
-          DBUG_RETURN(false);
-        }
+      if(err!= SCI_ERR_OK) {
+        DBUG_PRINT("error",
+          ("Cannot map a segment to the remote node %d. Error code 0x%x",
+          m_RemoteSciNodeId, err));
+        //NDB SHOULD TERMINATE AND COMPUTER REBOOTED! 
+        report_error(TE_SCI_CANNOT_MAP_REMOTESEGMENT);
+        DBUG_RETURN(false);
+      }
     }
     m_mapped=true;
     setupRemoteSegment();
@@ -713,7 +615,6 @@ SCI_Transporter::connect_client_impl(NDB_SOCKET_TYPE sockfd)
     NDB_CLOSE_SOCKET(sockfd);
     DBUG_RETURN(false);
   }
-
   if (!init_local()) {
     NDB_CLOSE_SOCKET(sockfd);
     DBUG_RETURN(false);
@@ -788,29 +689,9 @@ sci_error_t SCI_Transporter::createSequence(Uint32 adapterid) {
 		       &(m_TargetSegm[adapterid].sequence),  
 		       SCI_FLAG_FAST_BARRIER,  
 		       &err);  
-  
-  
   return err; 
 } // createSequence()  
  
- 
-sci_error_t SCI_Transporter::startSequence(Uint32 adapterid) { 
-  
-  sci_error_t err; 
-  /** Perform preliminary error check on an SCI adapter before starting a 
-   * sequence of read and write operations on the mapped segment. 
-   */ 
-  m_SequenceStatus = SCIStartSequence( 
-				       (m_TargetSegm[adapterid].sequence),  
-				       FLAGS, &err); 
-   
-   
-  // If there still is an error then data cannot be safely send 
-  return err; 
-} // startSequence() 
- 
-   
- 
 bool SCI_Transporter::disconnectLocal()  
 {
   DBUG_ENTER("SCI_Transporter::disconnectLocal"); 
@@ -878,9 +759,6 @@ SCI_Transporter::~SCI_Transporter() {
   DBUG_VOID_RETURN;
 } // ~SCI_Transporter() 
  
- 
- 
- 
 void SCI_Transporter::closeSCI() { 
   // Termination of SCI 
   sci_error_t err; 
@@ -897,8 +775,9 @@ void SCI_Transporter::closeSCI() {
   SCIClose(activeSCIDescriptor, FLAGS, &err);  
    
   if(err != SCI_ERR_OK) {
-    DBUG_PRINT("error", ("Cannot close SCI channel to the driver. Error code 0x%x",  
-	        err)); 
+    DBUG_PRINT("error",
+      ("Cannot close SCI channel to the driver. Error code 0x%x",  
+      err)); 
   }
   SCITerminate(); 
   DBUG_VOID_RETURN;
@@ -973,7 +852,6 @@ SCI_Transporter::getConnectionStatus() {
     return false; 
 } 
  
- 
 void  
 SCI_Transporter::setConnected() { 
   *m_remoteStatusFlag = SCICONNECTED; 
@@ -983,7 +861,6 @@ SCI_Transporter::setConnected() {
   *m_localStatusFlag = SCICONNECTED; 
 } 
  
- 
 void  
 SCI_Transporter::setDisconnect() { 
   if(getLinkStatus(m_ActiveAdapterId)) 
@@ -994,7 +871,6 @@ SCI_Transporter::setDisconnect() {
   }
 } 
  
- 
 bool 
 SCI_Transporter::checkConnected() { 
   if (*m_localStatusFlag == SCIDISCONNECT) { 
@@ -1015,8 +891,9 @@ SCI_Transporter::initSCI() {
     SCIInitialize(0, &error); 
     if(error != SCI_ERR_OK)  { 
       DBUG_PRINT("error", ("Cannot initialize SISCI library."));
-      DBUG_PRINT("error", ("Inconsistency between SISCI library and SISCI driver. Error code 0x%x",
-                 error)); 
+      DBUG_PRINT("error",
+      ("Inconsistency between SISCI library and SISCI driver. Error code 0x%x",
+      error)); 
       DBUG_RETURN(false);
     } 
     init = true; 
@@ -1029,3 +906,4 @@ SCI_Transporter::get_free_buffer() const
 {
   return (m_TargetSegm[m_ActiveAdapterId].writer)->get_free_buffer();
 }
+
diff --git a/storage/ndb/src/common/transporter/SCI_Transporter.hpp b/storage/ndb/src/common/transporter/SCI_Transporter.hpp
index fbba2ac4516..f774186f238 100644
--- a/storage/ndb/src/common/transporter/SCI_Transporter.hpp
+++ b/storage/ndb/src/common/transporter/SCI_Transporter.hpp
@@ -54,12 +54,12 @@
  *  local segment, the SCI transporter connects to a segment created by another 
  *  transporter at a remote node, and the maps the remote segment into its  
  *  virtual address space. However, since NDB Cluster relies on redundancy 
- *  at the network level, by using dual SCI adapters communica 
- * 
+ *  at the network level, by using dual SCI adapters communication can be
+ *  maintained even if one of the adapter cards fails (or anything on the
+ *  network this adapter card exists in e.g. an SCI switch failure).
  * 
  */ 
 
-
 /**  
  * class SCITransporter 
  * @brief - main class for the SCI transporter. 
@@ -84,16 +84,6 @@ public:
   sci_error_t createSequence(Uint32 adapterid);      
    
    
-  /** 
-   * starts a sequence for error checking. 
-   * The actual checking that a sequence is correct is done implicitly 
-   * in SCIMemCpy (in doSend).  
-   * @param adapterid the adapter on which to start the sequence. 
-   * @return SCI_ERR_OK if ok, otherwize something else. 
-   */ 
-  sci_error_t startSequence(Uint32 adapterid);          
- 
- 
   /** Initiate Local Segment: create a memory segment, 
    * prepare a memory segment, map the local segment  
    * into  memory space and make segment available. 
@@ -159,7 +149,6 @@ private:
   bool m_mapped; 
   bool m_initLocal; 
   bool m_sciinit; 
-  Uint32 m_swapCounter; 
   Uint32 m_failCounter; 
   /** 
    * For statistics on transfered packets  
@@ -195,7 +184,6 @@ private:
    */ 
   Uint32 m_reportFreq; 
  
- 
   Uint32 m_adapters;   
   Uint32 m_numberOfRemoteNodes;