- Initial PySMI integration. Original ASN.1 MIBs could now be parsed, stored

  at a local pysnmp MIBs repository and loaded into SNMP Engine.
- smi.MibBuilder will now raise more specific exceptions (MibLoadError,
  MibNotFoundError) on MIB loading problems rather than more generic
  SmiError.
- MibBuilder.addMibSources() convenience method added.

7 files changed, 204 insertions, 65 deletions

diff --git a/CHANGES b/CHANGES
index f2a85a7..cecd2d2 100644
--- a/CHANGES
+++ b/CHANGES
@@ -3,6 +3,8 @@ Revision 4.2.6rc2
 
 - Critical error fixed in key localization procedure for AES192/AES256/3DES
   cyphers. Previous versions might never worked properly in this respect.
+- Initial PySMI integration. Original ASN.1 MIBs could now be parsed, stored
+  at a local pysnmp MIBs repository and loaded into SNMP Engine.
 - Major rewrite of native SNMPv3 CommandGenerator and NotificationOriginator
   applications towards the following goals:
   * avoid bonding with particular SNMP engine instance to promote single
@@ -80,8 +82,10 @@ Revision 4.2.6rc2
 - Parts of SMIv1 remnant MIBs (RFC1213-MIB, RFC1158-MIB) added to provide 
   complete compatibility with SMIv1. Symbols defined in these MIBs only
   present in SMIv1 so they can't be substituted with their SMIv2 analogues.
-- Optional configuration/MIB directory added to MIB search path
-  (~/,pysnmp/mibs) at MibBuilder.
+- MibBuilder.addMibSources() convenience method added.
+- The smi.MibBuilder() will now raise more specific exceptions (MibLoadError,
+  MibNotFoundError) on MIB loading problems rather than more generic
+  SmiError.
 - Fix to authoritative engine side snmpEngineID discovery procedure: 
   respond with notInTimeWindows rather then with unsupportedSecurityLevel
   at time synchronization phase.
diff --git a/examples/smi/view.py b/examples/smi/view.py
index e0134c2..549ea36 100644
--- a/examples/smi/view.py
+++ b/examples/smi/view.py
@@ -1,21 +1,23 @@
 # SNMP manager-side MIB management
-from pysnmp.smi import builder, view, error
+from pysnmp.smi import builder, view, compiler, error
 
 # Create MIB loader/builder
 mibBuilder = builder.MibBuilder()
 
-# Optionally set an alternative path to compiled MIBs
-#print('Setting MIB sources...')
-#mibSources = mibBuilder.getMibSources() + (
-#    builder.DirMibSource('/opt/pysnmp_mibs'),
-#    )
-#mibBuilder.setMibSources(*mibSources)
-#print(mibBuilder.getMibSources())
+# Optionally attach PySMI MIB compiler (if installed)
+#print('Attaching MIB compiler...'),
+#compiler.addMibCompiler(mibBuilder, sources=['/usr/share/snmp/mibs'])
 #print('done')
 
+# Optionally set an alternative path to compiled MIBs
+print('Setting MIB sources...')
+mibBuilder.addMibSources(builder.DirMibSource('/opt/pysnmp_mibs'))
+print(mibBuilder.getMibSources())
+print('done')
+
 print('Loading MIB modules...'),
 mibBuilder.loadModules(
-    'SNMPv2-MIB', 'SNMP-FRAMEWORK-MIB', 'SNMP-COMMUNITY-MIB'
+    'SNMPv2-MIB', 'SNMP-FRAMEWORK-MIB', 'SNMP-COMMUNITY-MIB', 'IP-MIB'
     )
 print('done')
 
diff --git a/examples/v3arch/oneliner/manager/cmdgen/get-v2c-with-asn1-mib-lookup.py b/examples/v3arch/oneliner/manager/cmdgen/get-v2c-with-asn1-mib-lookup.py
new file mode 100644
index 0000000..a8f1e73
--- /dev/null
+++ b/examples/v3arch/oneliner/manager/cmdgen/get-v2c-with-asn1-mib-lookup.py
@@ -0,0 +1,37 @@
+#
+# Command Generator
+#
+# Send SNMP GET request using the following options:
+#
+# * with SNMPv2c, community 'public'
+# * over IPv4/UDP
+# * to an Agent at demo.snmplabs.com:161
+# * for IF-MIB::ifInOctets.1 MIB object
+#
+from pysnmp.entity.rfc3413.oneliner import cmdgen
+from pysnmp import debug
+
+#debug.setLogger(debug.Debug('mibbuild'))
+
+cmdGen = cmdgen.CommandGenerator()
+
+errorIndication, errorStatus, errorIndex, varBinds = cmdGen.getCmd(
+    cmdgen.CommunityData('public'),
+    cmdgen.UdpTransportTarget(('demo.snmplabs.com', 161)),
+    cmdgen.MibVariable('IF-MIB', 'ifInOctets', '1').addAsn1Sources('file:///usr/share/snmp', 'http://mibs.snmplabs.com/asn1/<mib>'),
+    lookupNames=True, lookupValues=True
+)
+
+# Check for errors and print out results
+if errorIndication:
+    print(errorIndication)
+else:
+    if errorStatus:
+        print('%s at %s' % (
+            errorStatus.prettyPrint(),
+            errorIndex and varBinds[int(errorIndex)-1][0] or '?'
+            )
+        )
+    else:
+        for name, val in varBinds:
+            print('%s = %s' % (name.prettyPrint(), val.prettyPrint()))
diff --git a/pysnmp/entity/rfc3413/oneliner/mibvar.py b/pysnmp/entity/rfc3413/oneliner/mibvar.py
index 6846334..cb7b163 100644
--- a/pysnmp/entity/rfc3413/oneliner/mibvar.py
+++ b/pysnmp/entity/rfc3413/oneliner/mibvar.py
@@ -1,5 +1,6 @@
 from pysnmp.proto import rfc1902
 from pysnmp.smi.builder import ZipMibSource
+from pysnmp.smi.compiler import addMibCompiler
 from pysnmp.error import PySnmpError
 from pyasn1.error import PyAsn1Error
 
@@ -20,6 +21,7 @@ class MibVariable:
     def __init__(self, *args):
         self.__args = args
         self.__mibSourcesToAdd = self.__modNamesToLoad = None
+        self.__asn1SourcesToAdd = None
         self.__state  = self.stDirty
 
     #
@@ -56,6 +58,10 @@ class MibVariable:
     # A gateway to MIBs manipulation routines
     #
 
+    def addAsn1Sources(self, *asn1Sources):
+        self.__asn1SourcesToAdd = asn1Sources
+        return self
+
     def addMibSource(self, *mibSources):
         self.__mibSourcesToAdd = mibSources
         return self
@@ -75,6 +81,13 @@ class MibVariable:
             mibViewController.mibBuilder.setMibSources(*mibSources)
             self.__mibSourcesToAdd = None
 
+        if self.__asn1SourcesToAdd is not None:
+            addMibCompiler(
+                mibViewController.mibBuilder,
+                sources=self.__asn1SourcesToAdd
+            )
+            self.__asn1SourcesToAdd = None
+
         if self.__modNamesToLoad is not None:
             mibViewController.mibBuilder.loadModules(*self.__modNamesToLoad)
             self.__modNamesToLoad = None
diff --git a/pysnmp/smi/builder.py b/pysnmp/smi/builder.py
index 610e89d..795da83 100644
--- a/pysnmp/smi/builder.py
+++ b/pysnmp/smi/builder.py
@@ -200,17 +200,7 @@ class MibBuilder:
     defaultCoreMibs = os.pathsep.join(
         ('pysnmp.smi.mibs.instances', 'pysnmp.smi.mibs')
     )
-    if sys.platform[:3] == 'win':
-        defaultMiscMibs = os.pathsep.join(
-            ( os.path.join(os.path.expanduser("~"),
-                           'PySNMP Configuration', 'mibs'),
-              'pysnmp_mibs' )
-        )
-    else:
-        defaultMiscMibs = os.pathsep.join(
-            ( os.path.join(os.path.expanduser("~"), '.pysnmp', 'mibs'),
-              'pysnmp_mibs' )
-        )
+    defaultMiscMibs = 'pysnmp_mibs'
     moduleID = 'PYSNMP_MODULE_ID'
     def __init__(self):
         self.lastBuildId = self._autoName = 0
@@ -229,13 +219,29 @@ class MibBuilder:
         self.mibSymbols = {}
         self.__modSeen = {}
         self.__modPathsSeen = {}
+        self.__mibCompiler = None
         self.setMibSources(*sources)
-        
+
+    # MIB compiler management
+
+    def getMibCompiler(self):
+        return self.__mibCompiler
+       
+    def setMibCompiler(self, mibCompiler, destDir):
+        self.addMibSources(DirMibSource(destDir))
+        self.__mibCompiler = mibCompiler
+        return self
+       
     # MIB modules management
 
+    def addMibSources(self, *mibSources):
+        self.__mibSources.extend([ s.init() for s in mibSources ])
+        debug.logger & debug.flagBld and debug.logger('addMibSources: new MIB sources %s' % (self.__mibSources,))
+
+
     def setMibSources(self, *mibSources):
         self.__mibSources = [ s.init() for s in mibSources ]
-        debug.logger & debug.flagBld and debug.logger('setMibPath: new MIB sources %s' % (self.__mibSources,))
+        debug.logger & debug.flagBld and debug.logger('setMibSources: new MIB sources %s' % (self.__mibSources,))
 
     def getMibSources(self): return tuple(self.__mibSources)
 
@@ -254,6 +260,49 @@ class MibBuilder:
                     )
         return paths
         
+    def loadModule(self, modName, **userCtx):
+        for mibSource in self.__mibSources:
+            debug.logger & debug.flagBld and debug.logger('loadModule: trying %s at %s' % (modName, mibSource))
+            try:
+                modData, sfx = mibSource.read(modName)
+            except IOError:
+                debug.logger & debug.flagBld and debug.logger('loadModule: read %s from %s failed: %s' % (modName, mibSource, sys.exc_info()[1]))
+                continue
+
+            modPath = mibSource.fullPath(modName, sfx)
+            
+            if modPath in self.__modPathsSeen:
+                debug.logger & debug.flagBld and debug.logger('loadModule: seen %s' % modPath)
+                break
+            else:
+                self.__modPathsSeen[modPath] = 1
+
+            debug.logger & debug.flagBld and debug.logger('loadModule: evaluating %s' % modPath)
+
+            g = { 'mibBuilder': self,
+                  'userCtx': userCtx }
+
+            try:
+                exec(modData, g)
+            except Exception:
+                del self.__modPathsSeen[modPath]
+                raise error.MibLoadError(
+                    'MIB module \"%s\" load error: %s' % (modPath, traceback.format_exception(*sys.exc_info()))
+                    )
+
+            self.__modSeen[modName] = modPath
+
+            debug.logger & debug.flagBld and debug.logger('loadModule: loaded %s' % modPath)
+
+            break
+
+        if modName not in self.__modSeen:
+            raise error.MibNotFoundError(
+                'MIB file \"%s\" not found in search path (%s)' % (modName and modName + ".py[co]", ', '.join([str(x) for x in self.__mibSources]))
+            )
+
+        return self
+
     def loadModules(self, *modNames, **userCtx):
         # Build a list of available modules
         if not modNames:
@@ -263,50 +312,19 @@ class MibBuilder:
                     modNames[modName] = None
             modNames = list(modNames.keys())
         if not modNames:
-            raise error.SmiError(
+            raise error.MibNotFoundError(
                 'No MIB module to load at %s' % (self,)
-                )
+            )
         
         for modName in modNames:
-            for mibSource in self.__mibSources:
-                debug.logger & debug.flagBld and debug.logger('loadModules: trying %s at %s' % (modName, mibSource))
-                try:
-                    modData, sfx = mibSource.read(modName)
-                except IOError:
-                    debug.logger & debug.flagBld and debug.logger('loadModules: read %s from %s failed: %s' % (modName, mibSource, sys.exc_info()[1]))
-                    continue
-
-                modPath = mibSource.fullPath(modName, sfx)
-                
-                if modPath in self.__modPathsSeen:
-                    debug.logger & debug.flagBld and debug.logger('loadModules: seen %s' % modPath)
-                    break
-                else:
-                    self.__modPathsSeen[modPath] = 1
-
-                debug.logger & debug.flagBld and debug.logger('loadModules: evaluating %s' % modPath)
-
-                g = { 'mibBuilder': self,
-                      'userCtx': userCtx }
-
-                try:
-                    exec(modData, g)
-                except Exception:
-                    del self.__modPathsSeen[modPath]
-                    raise error.SmiError(
-                        'MIB module \"%s\" load error: %s' % (modPath, traceback.format_exception(*sys.exc_info()))
-                        )
-
-                self.__modSeen[modName] = modPath
-
-                debug.logger & debug.flagBld and debug.logger('loadModules: loaded %s' % modPath)
-
-                break
-
-            if modName not in self.__modSeen:
-                raise error.SmiError(
-                    'MIB file \"%s\" not found in search path (%s)' % (modName and modName + ".py[co]", ', '.join([str(x) for x in self.__mibSources]))
-                    )
+            try:
+                self.loadModule(modName, **userCtx)
+            except error.MibNotFoundError:
+                if self.__mibCompiler:
+                    debug.logger & debug.flagBld and debug.logger('loadModules: calling MIB compiler for %s' % modName)
+                    self.__mibCompiler.compile(modName)
+                    # in case this missing MIB becomes available
+                    self.loadModule(modName, **userCtx)
 
         return self
                 
diff --git a/pysnmp/smi/compiler.py b/pysnmp/smi/compiler.py
new file mode 100644
index 0000000..80e041c
--- /dev/null
+++ b/pysnmp/smi/compiler.py
@@ -0,0 +1,62 @@
+#
+# Attach PySMI MIB compiler to PySNMP MIB builder and configure 
+# both accordingly.
+#
+import os
+import sys
+try:
+    from pysmi.reader.url import getReadersFromUrls
+    from pysmi.searcher.pypackage import PyPackageSearcher
+    from pysmi.searcher.stub import StubSearcher
+    from pysmi.borrower.pyfile import PyFileBorrower
+    from pysmi.writer.pyfile import PyFileWriter
+    from pysmi.parser.smi import parserFactory
+    from pysmi.parser.dialect import smiV1Relaxed
+    from pysmi.codegen.pysnmp import PySnmpCodeGen, baseMibs
+    from pysmi.compiler import MibCompiler
+
+except ImportError:
+    from pysnmp.smi import error
+
+    def addMibCompiler(mibBuilder,
+                       sources=defaultSources,
+                       destination=defaultDest,
+                       borrowers=defaultBorrowers):
+        raise error.SmiError('MIB compiler not available (pysmi not installed)')
+
+else:
+    defaultSources = [ 'file:///usr/share/snmp/mibs' ]
+
+    if sys.platform[:3] == 'win':
+        defaultDest = os.path.join(os.path.expanduser("~"),
+                                   'PySNMP Configuration', 'mibs')
+    else:
+        defaultDest = os.path.join(os.path.expanduser("~"), '.pysnmp', 'mibs')
+
+    defaultBorrowers = []
+
+    def addMibCompiler(mibBuilder,
+                       sources=defaultSources,
+                       destination=defaultDest,
+                       borrowers=defaultBorrowers):
+
+        compiler = MibCompiler(
+            parserFactory(**smiV1Relaxed)(),
+            PySnmpCodeGen(),
+            PyFileWriter(destination)
+        )
+
+        compiler.addSources(*getReadersFromUrls(*sources))
+
+        compiler.addSearchers(
+            StubSearcher(*baseMibs) # XXX
+        )
+        compiler.addSearchers(
+            *[ PyPackageSearcher(x.fullPath()) for x in mibBuilder.getMibSources() ]
+        )
+
+        compiler.addBorrowers(
+            *[ PyFileBorrower(x) for x in getReadersFromUrls(*borrowers, originalMatching=False, lowcaseMatching=False) ]
+        )
+
+        mibBuilder.setMibCompiler(compiler, destination)
diff --git a/pysnmp/smi/error.py b/pysnmp/smi/error.py
index cb0de11..1895a97 100644
--- a/pysnmp/smi/error.py
+++ b/pysnmp/smi/error.py
@@ -2,6 +2,9 @@ from pyasn1.error import PyAsn1Error
 from pysnmp.error import PySnmpError
 
 class SmiError(PySnmpError, PyAsn1Error): pass
+class MibLoadError(SmiError): pass
+class MibNotFoundError(MibLoadError): pass
+
 class MibOperationError(SmiError):
     def __init__(self, **kwargs): self.__outArgs = kwargs
     def __str__(self): return '%s(%s)' % (