Execution Observer facility implemented to give app an inside view

of SNMP engine inner workings. This is thought to be a generic
framework for viewing (and modifying) various internal states
of pysnmp engine. Previously introduced non-RFC APIs (like
getting peer's transport endpoint) will be gradually migrated to
this new framework.

10 files changed, 385 insertions, 50 deletions

diff --git a/examples/v3arch/agent/cmdrsp/v3-observe-request-processing.py b/examples/v3arch/agent/cmdrsp/v3-observe-request-processing.py
new file mode 100644
index 0000000..fec2a6a
--- /dev/null
+++ b/examples/v3arch/agent/cmdrsp/v3-observe-request-processing.py
@@ -0,0 +1,84 @@
+#
+# Command Responder
+#
+# Listen and respond to SNMP GET/SET/GETNEXT/GETBULK queries with
+# the following options:
+#
+# * SNMPv3
+# * with USM user 'usr-md5-des', auth: MD5, priv DES or
+# * allow access to SNMPv2-MIB objects (1.3.6.1.2.1)
+# * over IPv4/UDP, listening at 127.0.0.1:161
+# * registers its own execution observer to snmpEngine
+# 
+# Either of the following Net-SNMP's commands will walk this Agent:
+#
+# $ snmpwalk -v3 -u usr-md5-des -l authPriv -A authkey1 -X privkey1 localhost .1.3.6
+#
+from pysnmp.entity import engine, config
+from pysnmp.entity.rfc3413 import cmdrsp, context
+from pysnmp.carrier.asynsock.dgram import udp
+
+# Create SNMP engine
+snmpEngine = engine.SnmpEngine()
+
+# Execution point observer setup
+
+# Register a callback to be invoked at specified execution point of 
+# SNMP Engine and passed local variables at code point's local scope
+def requestObserver(snmpEngine, execpoint, variables, cbCtx):
+    print('Execution point: %s' % execpoint)
+    print('* transportDomain: %s' % '.'.join([str(x) for x in variables['transportDomain']]))
+    print('* transportAddress: %s' % '@'.join([str(x) for x in variables['transportAddress']]))
+    print('* securityModel: %s' % variables['securityModel'])
+    print('* securityName: %s' % variables['securityName'])
+    print('* securityLevel: %s' % variables['securityLevel'])
+    print('* contextEngineId: %s' % variables['contextEngineId'].prettyPrint())
+    print('* contextName: %s' % variables['contextName'].prettyPrint())
+    print('* PDU: %s' % variables['pdu'].prettyPrint())
+
+snmpEngine.observer.registerObserver(
+    requestObserver,
+    'rfc3412.receiveMessage:request',
+    'rfc3412.returnResponsePdu'
+)
+
+# Transport setup
+
+# UDP over IPv4
+config.addTransport(
+    snmpEngine,
+    udp.domainName,
+    udp.UdpTransport().openServerMode(('127.0.0.1', 161))
+)
+
+# SNMPv3/USM setup
+
+# user: usr-md5-des, auth: MD5, priv DES
+config.addV3User(
+    snmpEngine, 'usr-md5-des',
+    config.usmHMACMD5AuthProtocol, 'authkey1',
+    config.usmDESPrivProtocol, 'privkey1'
+)
+
+# Allow full MIB access for each user at VACM
+config.addVacmUser(snmpEngine, 3, 'usr-md5-des', 'authPriv', (1,3,6,1,2,1), (1,3,6,1,2,1)) 
+
+# Get default SNMP context this SNMP engine serves
+snmpContext = context.SnmpContext(snmpEngine)
+
+# Register SNMP Applications at the SNMP engine for particular SNMP context
+cmdrsp.GetCommandResponder(snmpEngine, snmpContext)
+cmdrsp.SetCommandResponder(snmpEngine, snmpContext)
+cmdrsp.NextCommandResponder(snmpEngine, snmpContext)
+cmdrsp.BulkCommandResponder(snmpEngine, snmpContext)
+
+# Register an imaginary never-ending job to keep I/O dispatcher running forever
+snmpEngine.transportDispatcher.jobStarted(1)
+
+# Run I/O dispatcher which would receive queries and send responses
+try:
+    snmpEngine.transportDispatcher.runDispatcher()
+except:
+    snmpEngine.observer.unregisterObserver()
+    snmpEngine.transportDispatcher.closeDispatcher()
+    raise
diff --git a/examples/v3arch/manager/cmdgen/get-v3-observe-request-processing.py b/examples/v3arch/manager/cmdgen/get-v3-observe-request-processing.py
new file mode 100644
index 0000000..9b8d7dc
--- /dev/null
+++ b/examples/v3arch/manager/cmdgen/get-v3-observe-request-processing.py
@@ -0,0 +1,99 @@
+#
+# GET Command Generator
+#
+# Send a SNMP GET request
+#     with SNMPv3 with user 'usr-sha-aes', SHA auth and AES128 privacy protocols
+#     over IPv4/UDP
+#     to an Agent at 195.218.195.228:161
+#     for an OID in tuple form
+#     also registers its own execution observer to snmpEngine
+#
+# This script performs similar to the following Net-SNMP command:
+#
+# $ snmpget -v3 -l authPriv -u usr-sha-aes -a SHA -A authkey1 -x AES -X privkey1 -ObentU 195.218.195.228:161  1.3.6.1.2.1.1.1.0
+#
+from pysnmp.entity import engine, config
+from pysnmp.carrier.asynsock.dgram import udp
+from pysnmp.entity.rfc3413 import cmdgen
+
+# Create SNMP engine instance
+snmpEngine = engine.SnmpEngine()
+
+# Execution point observer setup
+
+# Register a callback to be invoked at specified execution point of 
+# SNMP Engine and passed local variables at code point's local scope
+def requestObserver(snmpEngine, execpoint, variables, cbCtx):
+    print('Execution point: %s' % execpoint)
+    print('* transportDomain: %s' % '.'.join([str(x) for x in variables['transportDomain']]))
+    print('* transportAddress: %s' % '@'.join([str(x) for x in variables['transportAddress']]))
+    print('* securityModel: %s' % variables['securityModel'])
+    print('* securityName: %s' % variables['securityName'])
+    print('* securityLevel: %s' % variables['securityLevel'])
+    print('* contextEngineId: %s' % (variables['contextEngineId'] and variables['contextEngineId'].prettyPrint() or '<empty>',))
+    print('* contextName: %s' % variables['contextName'].prettyPrint())
+    print('* PDU: %s' % variables['pdu'].prettyPrint())
+
+snmpEngine.observer.registerObserver(
+    requestObserver, 
+    'rfc3412.sendPdu',
+    'rfc3412.receiveMessage:response'
+)
+
+#
+# SNMPv3/USM setup
+#
+
+# user: usr-sha-aes, auth: SHA, priv AES
+config.addV3User(
+    snmpEngine, 'usr-sha-aes',
+    config.usmHMACSHAAuthProtocol, 'authkey1',
+    config.usmAesCfb128Protocol, 'privkey1'
+)
+config.addTargetParams(snmpEngine, 'my-creds', 'usr-sha-aes', 'authPriv')
+
+#
+# Setup transport endpoint and bind it with security settings yielding
+# a target name
+#
+
+# UDP/IPv4
+config.addTransport(
+    snmpEngine,
+    udp.domainName,
+    udp.UdpSocketTransport().openClientMode()
+)
+config.addTargetAddr(
+    snmpEngine, 'my-router',
+    udp.domainName, ('195.218.195.228', 161),
+    'my-creds'
+)
+
+# Error/response receiver
+def cbFun(sendRequestHandle,
+          errorIndication, errorStatus, errorIndex,
+          varBinds, cbCtx):
+    if errorIndication:
+        print(errorIndication)
+    elif errorStatus:
+        print('%s at %s' % (
+            errorStatus.prettyPrint(),
+            errorIndex and varBinds[int(errorIndex)-1][0] or '?'
+            )
+        )
+    else:
+        for oid, val in varBinds:
+            print('%s = %s' % (oid.prettyPrint(), val.prettyPrint()))
+
+# Prepare and send a request message
+cmdgen.GetCommandGenerator().sendReq(
+    snmpEngine,
+    'my-router',
+    ( ((1,3,6,1,2,1,1,1,0), None), ),
+    cbFun
+)
+
+# Run I/O dispatcher which would send pending queries and process responses
+snmpEngine.transportDispatcher.runDispatcher()
+
+snmpEngine.observer.unregisterObserver()
diff --git a/examples/v3arch/manager/ntfrcv/v2c-multiple-interfaces.py b/examples/v3arch/manager/ntfrcv/v2c-multiple-interfaces.py
index 2a923e0..182338b 100644
--- a/examples/v3arch/manager/ntfrcv/v2c-multiple-interfaces.py
+++ b/examples/v3arch/manager/ntfrcv/v2c-multiple-interfaces.py
@@ -45,14 +45,10 @@ config.addTransport(
 config.addV1System(snmpEngine, 'my-area', 'public')
 
 # Callback function for receiving notifications
-def cbFun(snmpEngine,
-          stateReference,
-          contextEngineId, contextName,
-          varBinds,
-          cbCtx):
-    transportDomain, transportAddress = snmpEngine.msgAndPduDsp.getTransportInfo(stateReference)
-    print('Notification from %s, ContextEngineId "%s", ContextName "%s"' % (
-        transportAddress, contextEngineId.prettyPrint(),
+def cbFun(snmpEngine, stateReference, contextEngineId, contextName,
+          varBinds, cbCtx):
+    print('Notification from ContextEngineId "%s", ContextName "%s"' % (
+        contextEngineId.prettyPrint(),
         contextName.prettyPrint()
         )
     )
diff --git a/examples/v3arch/manager/ntfrcv/v2c-multiple-transports.py b/examples/v3arch/manager/ntfrcv/v2c-multiple-transports.py
index 7b0a2cd..1f9c739 100644
--- a/examples/v3arch/manager/ntfrcv/v2c-multiple-transports.py
+++ b/examples/v3arch/manager/ntfrcv/v2c-multiple-transports.py
@@ -46,14 +46,10 @@ config.addTransport(
 config.addV1System(snmpEngine, 'my-area', 'public')
 
 # Callback function for receiving notifications
-def cbFun(snmpEngine,
-          stateReference,
-          contextEngineId, contextName,
-          varBinds,
-          cbCtx):
-    transportDomain, transportAddress = snmpEngine.msgAndPduDsp.getTransportInfo(stateReference)
-    print('Notification from %s, ContextEngineId "%s", ContextName "%s"' % (
-        transportAddress, contextEngineId.prettyPrint(),
+def cbFun(snmpEngine, stateReference, contextEngineId, contextName,
+          varBinds, cbCtx):
+    print('Notification from ContextEngineId "%s", ContextName "%s"' % (
+        contextEngineId.prettyPrint(),
         contextName.prettyPrint()
         )
     )
diff --git a/examples/v3arch/manager/ntfrcv/v2c-observe-request-processing.py b/examples/v3arch/manager/ntfrcv/v2c-observe-request-processing.py
new file mode 100644
index 0000000..41796dd
--- /dev/null
+++ b/examples/v3arch/manager/ntfrcv/v2c-observe-request-processing.py
@@ -0,0 +1,92 @@
+#
+# Notification Receiver
+#
+# Receive SNMP TRAP/INFORM messages with the following options:
+#
+# * SNMPv1/SNMPv2c
+# * with SNMP community "public"
+# * over IPv4/UDP, listening at 127.0.0.1:162
+#   over IPv6/UDP, listening at [::1]:162
+# * registers its own execution observer to snmpEngine
+# * print received data on stdout
+# 
+# Either of the following Net-SNMP's commands will send notifications to this
+# receiver:
+#
+# $ snmptrap -v1 -c public 127.0.0.1 1.3.6.1.4.1.20408.4.1.1.2 127.0.0.1 1 1 123 1.3.6.1.2.1.1.1.0 s test
+# $ snmptrap -v2c -c public udp6:[::1]:162 123 1.3.6.1.6.3.1.1.5.1 1.3.6.1.2.1.1.5.0 s test
+# $ snmpinform -v2c -c public 127.0.0.1 123 1.3.6.1.6.3.1.1.5.1
+#
+from pysnmp.entity import engine, config
+from pysnmp.carrier.asynsock.dgram import udp, udp6
+from pysnmp.entity.rfc3413 import ntfrcv
+
+# Create SNMP engine with autogenernated engineID and pre-bound
+# to socket transport dispatcher
+snmpEngine = engine.SnmpEngine()
+
+# Execution point observer setup
+
+# Register a callback to be invoked at specified execution point of 
+# SNMP Engine and passed local variables at code point's local scope
+def requestObserver(snmpEngine, execpoint, variables, cbCtx):
+    print('Execution point: %s' % execpoint)
+    print('* transportDomain: %s' % '.'.join([str(x) for x in variables['transportDomain']]))
+    print('* transportAddress: %s' % '@'.join([str(x) for x in variables['transportAddress']]))
+    print('* securityModel: %s' % variables['securityModel'])
+    print('* securityName: %s' % variables['securityName'])
+    print('* securityLevel: %s' % variables['securityLevel'])
+    print('* contextEngineId: %s' % variables['contextEngineId'].prettyPrint())
+    print('* contextName: %s' % variables['contextName'].prettyPrint())
+    print('* PDU: %s' % variables['pdu'].prettyPrint())
+
+snmpEngine.observer.registerObserver(
+    requestObserver,
+    'rfc3412.receiveMessage:request',
+    'rfc3412.returnResponsePdu'
+)
+
+# Transport setup
+
+# UDP over IPv4
+config.addTransport(
+    snmpEngine,
+    udp.domainName,
+    udp.UdpTransport().openServerMode(('127.0.0.1', 162))
+)
+
+# UDP over IPv6
+config.addTransport(
+    snmpEngine,
+    udp6.domainName,
+    udp6.Udp6Transport().openServerMode(('::1', 162))
+    )
+
+# SNMPv1/2c setup
+
+# SecurityName <-> CommunityName mapping
+config.addV1System(snmpEngine, 'my-area', 'public')
+
+# Callback function for receiving notifications
+def cbFun(snmpEngine, stateReference, contextEngineId, contextName,
+          varBinds, cbCtx):
+    print('Notification from ContextEngineId "%s", ContextName "%s"' % (
+        contextEngineId.prettyPrint(),
+        contextName.prettyPrint()
+        )
+    )
+    for name, val in varBinds:
+        print('%s = %s' % (name.prettyPrint(), val.prettyPrint()))
+
+# Register SNMP Application at the SNMP engine
+ntfrcv.NotificationReceiver(snmpEngine, cbFun)
+
+snmpEngine.transportDispatcher.jobStarted(1) # this job would never finish
+
+# Run I/O dispatcher which would receive queries and send confirmations
+try:
+    snmpEngine.transportDispatcher.runDispatcher()
+except:
+    snmpEngine.observer.unregisterObserver()
+    snmpEngine.transportDispatcher.closeDispatcher()
+    raise
diff --git a/examples/v3arch/manager/ntfrcv/v2c-with-request-details.py b/examples/v3arch/manager/ntfrcv/v2c-with-request-details.py
new file mode 100644
index 0000000..e7f1301
--- /dev/null
+++ b/examples/v3arch/manager/ntfrcv/v2c-with-request-details.py
@@ -0,0 +1,67 @@
+#
+# Notification Receiver
+#
+# Receive SNMP TRAP/INFORM messages with the following options:
+#
+# * SNMPv1/SNMPv2c
+# * with SNMP community "public"
+# * over IPv4/UDP, listening at 127.0.0.1:162
+# * use observer facility to pull lower-level request details from SNMP engine
+# * print received data on stdout
+# 
+# Either of the following Net-SNMP's commands will send notifications to this
+# receiver:
+#
+# $ snmptrap -v2c -c public 127.0.0.1:162 123 1.3.6.1.6.3.1.1.5.1 1.3.6.1.2.1.1.5.0 s test
+#
+from pysnmp.entity import engine, config
+from pysnmp.carrier.asynsock.dgram import udp
+from pysnmp.entity.rfc3413 import ntfrcv
+
+# Create SNMP engine with autogenernated engineID and pre-bound
+# to socket transport dispatcher
+snmpEngine = engine.SnmpEngine()
+
+# Transport setup
+
+# UDP over IPv4, first listening interface/port
+config.addTransport(
+    snmpEngine,
+    udp.domainName + (1,),
+    udp.UdpTransport().openServerMode(('127.0.0.1', 162))
+)
+
+# SNMPv1/2c setup
+
+# SecurityName <-> CommunityName mapping
+config.addV1System(snmpEngine, 'my-area', 'public')
+
+# Callback function for receiving notifications
+def cbFun(snmpEngine, stateReference, contextEngineId, contextName,
+          varBinds, cbCtx):
+    # Get an execution context...
+    execContext = snmpEngine.observer.getExecutionContext(
+        'rfc3412.receiveMessage:request'
+    )
+
+    # ... and use inner SNMP engine data to figure out peer address
+    print('Notification from %s, ContextEngineId "%s", ContextName "%s"' % (
+        '@'.join([str(x) for x in execContext['transportAddress']]),
+        contextEngineId.prettyPrint(),
+        contextName.prettyPrint()
+        )
+    )
+    for name, val in varBinds:
+        print('%s = %s' % (name.prettyPrint(), val.prettyPrint()))
+
+# Register SNMP Application at the SNMP engine
+ntfrcv.NotificationReceiver(snmpEngine, cbFun)
+
+snmpEngine.transportDispatcher.jobStarted(1) # this job would never finish
+
+# Run I/O dispatcher which would receive queries and send confirmations
+try:
+    snmpEngine.transportDispatcher.runDispatcher()
+except:
+    snmpEngine.transportDispatcher.closeDispatcher()
+    raise
diff --git a/examples/v3arch/manager/ntfrcv/v3-multiple-users.py b/examples/v3arch/manager/ntfrcv/v3-multiple-users.py
index c2a08b8..9f26839 100644
--- a/examples/v3arch/manager/ntfrcv/v3-multiple-users.py
+++ b/examples/v3arch/manager/ntfrcv/v3-multiple-users.py
@@ -84,13 +84,11 @@ config.addV3User(
 )
 
 # Callback function for receiving notifications
-def cbFun(snmpEngine,
-          stateReference,
-          contextEngineId, contextName,
-          varBinds,
-          cbCtx):
-    print('Notification received, ContextEngineId "%s", ContextName "%s"' % (
-        contextEngineId.prettyPrint(), contextName.prettyPrint()
+def cbFun(snmpEngine, stateReference, contextEngineId, contextName,
+          varBinds, cbCtx):
+    print('Notification from ContextEngineId "%s", ContextName "%s"' % (
+        contextEngineId.prettyPrint(),
+        contextName.prettyPrint()
         )
     )
     for name, val in varBinds:
diff --git a/examples/v3arch/oneliner/manager/cmdgen/get-v3-with-specific-engine-id.py b/examples/v3arch/oneliner/manager/cmdgen/get-v3-with-specific-engine-id.py
index a59423f..efdcbc8 100644
--- a/examples/v3arch/oneliner/manager/cmdgen/get-v3-with-specific-engine-id.py
+++ b/examples/v3arch/oneliner/manager/cmdgen/get-v3-with-specific-engine-id.py
@@ -23,28 +23,38 @@ cmdGen = cmdgen.CommandGenerator(snmpEngine)
 transportTarget = cmdgen.UdpTransportTarget(('demo.snmplabs.com', 161))
 
 #
-# Discover remote SNMP EngineID
+# To discover remote SNMP EngineID we will tap on SNMP engine inner workings
+# by setting up execution point observer setup on INTERNAL class PDU processing
 #
 
+observerContext = {}
+
+# Register a callback to be invoked at specified execution point of 
+# SNMP Engine and passed local variables at execution point's local scope
+snmpEngine.observer.registerObserver(
+    lambda e,p,v,c: c.update(securityEngineId=v['securityEngineId']),
+    'rfc3412.prepareDataElements:internal',
+    cbCtx=observerContext
+)
+
+# Send probe SNMP request with invalid credentials
+
 authData = cmdgen.UsmUserData('non-existing-user')
 
 errorIndication, errorStatus, errorIndex, varBinds = cmdGen.getCmd(
     authData, transportTarget
 )
 
-# Check for errors and print out results
-if errorIndication == 'unknownUserName':
-    snmpV3MessageProcessor = snmpEngine.messageProcessingSubsystems[3]
-    securityEngineId, contextEngineId, contextName = snmpV3MessageProcessor.getPeerEngineInfo(*transportTarget.getTransportInfo())
-    if securityEngineId:
-        print('securityEngineId = %s' % securityEngineId.prettyPrint())
-    else:
-        print('Peer EngineID not available')
-        raise Exception()
-else:
-    print('Can\'t discover peer EngineID', errorIndication)
+# See if our SNMP engine received REPORT PDU containing securityEngineId
+
+if 'securityEngineId' not in observerContext:
+    print("Can't discover peer EngineID, errorIndication: %s" % errorIndication)
     raise Exception()
 
+securityEngineId = observerContext.pop('securityEngineId')
+
+print('Remote securityEngineId = %s' % securityEngineId.prettyPrint())
+
 #
 # Query remote SNMP Engine using usmUserTable entry configured for it
 #
diff --git a/examples/v3arch/twisted/manager/ntfrcv/v2c-multiple-interfaces.py b/examples/v3arch/twisted/manager/ntfrcv/v2c-multiple-interfaces.py
index f0edca7..4f9b68f 100644
--- a/examples/v3arch/twisted/manager/ntfrcv/v2c-multiple-interfaces.py
+++ b/examples/v3arch/twisted/manager/ntfrcv/v2c-multiple-interfaces.py
@@ -47,15 +47,11 @@ config.addTransport(
 config.addV1System(snmpEngine, 'my-area', 'public')
 
 # Callback function for receiving notifications
-def cbFun(snmpEngine,
-          stateReference,
-          contextEngineId, contextName,
-          varBinds,
-          cbCtx):
-    transportDomain, transportAddress = snmpEngine.msgAndPduDsp.getTransportInfo(stateReference)
-    print('Notification from %s, SNMP Engine %s, Context %s' % (
-            transportAddress, contextEngineId.prettyPrint(),
-            contextName.prettyPrint()
+def cbFun(snmpEngine, stateReference, contextEngineId, contextName,
+          varBinds, cbCtx):
+    print('Notification from ContextEngineId "%s", Context "%s"' % (
+        contextEngineId.prettyPrint(),
+        contextName.prettyPrint()
         )
     )
     for name, val in varBinds:
diff --git a/examples/v3arch/twisted/manager/ntfrcv/v3-multiple-users.py b/examples/v3arch/twisted/manager/ntfrcv/v3-multiple-users.py
index 6421672..1192427 100644
--- a/examples/v3arch/twisted/manager/ntfrcv/v3-multiple-users.py
+++ b/examples/v3arch/twisted/manager/ntfrcv/v3-multiple-users.py
@@ -89,12 +89,9 @@ config.addV3User(
 # -- end of SNMPv3/USM setup
 
 # Callback function for receiving notifications
-def cbFun(snmpEngine,
-          stateReference,
-          contextEngineId, contextName,
-          varBinds,
-          cbCtx):
-    print('Notification received, ContextEngineId "%s", ContextName "%s"' % (
+def cbFun(snmpEngine, stateReference, contextEngineId, contextName,
+          varBinds, cbCtx):
+    print('Notification from ContextEngineId "%s", ContextName "%s"' % (
             contextEngineId.prettyPrint(), contextName.prettyPrint()
         )
     )