initial commit, 'seems to work', README, notes

1 files changed, 226 insertions, 0 deletions

diff --git a/code/vlanscrape b/code/vlanscrape
new file mode 100755
index 0000000..25a05da
--- /dev/null
+++ b/code/vlanscrape
@@ -0,0 +1,226 @@
+#!/usr/bin/python
+#
+# 'router', 'host' and 'device' are used interchangably in this script
+
+configsDir   = "/store/rancid/nordunet/configs"
+outputDir    = "/home/vlanscrape/data"
+routerDBFile = "/store/rancid/nordunet/router.db"
+routerDB     = dict()
+
+# populate 'routerDB' with the contents of routerDBFile
+f = open(routerDBFile, 'r')
+for line in f:
+  hostname, vendor, state = line.split(";")
+  # ignore 'state' as it is of no consequence for the purpose of this script
+  routerDB[hostname]=vendor
+f.close()
+
+import os
+import errno
+import re
+import shutil
+import time
+
+# populate 'hostList'
+hostList = os.listdir(configsDir)
+if 'CVS' in hostList:
+  hostList.remove('CVS')
+if '.cvsignore' in hostList:
+  hostList.remove('.cvsignore')
+
+def is_arista(hostName):
+  vlanDict=dict()
+# print "arista:   ", hostName
+  inFile = configsDir + "/" + hostName
+  outFile = outputDir + "/" + hostName
+  f = open(inFile)
+  hostConfig=f.readlines()
+  f.close()
+  for i, l in enumerate(hostConfig):
+    if re.match("^vlan", hostConfig[i]):
+      # The config allows a comma separated list of multiple vlan ids,
+      # with the same name, and it also allows vlans to be unnamed.
+      # The simplest way to handle this is to grab the (lack of) name first
+      # and apply it to the (list of) vlan(s) after.
+      j = 1
+      while True:
+        if re.match(".*name ", hostConfig[i + j]):
+          name = re.sub(".*name ", "", hostConfig[i + j].rstrip())
+          break
+        elif re.match('^!', hostConfig[i + j]):
+          name = ""
+          break
+        else:
+          j += 1
+
+      vlan = re.sub(".*vlan ", "", hostConfig[i].rstrip())
+      if "," in vlan:
+        for v in vlan.split(","):
+          vlanDict[v] = name
+      elif "-" in vlan:
+        first, last = vlan.split("-")
+        for v in range(int(first),(int(last) + 1)):
+          vlanDict[v] = name
+      else:
+        vlanDict[vlan] = name
+
+  vlanList=vlanDict.items()
+  check_out_vs_dst(hostName,vlanList)
+
+def look_in_juniper_vlans_section(hostName, hostConfig):
+  # Since vlans in the 'vlans' section (of a switch config) can only exist
+  # once per 'device', using a dict here makes sense, as it ensures there will
+  # only be one instance.
+  vlanDict=dict()
+  vBegin = 0
+  vEnd = 0
+  for i, l in enumerate(hostConfig):
+    if re.match('^vlans {', l):
+      vBegin = i
+    if (vBegin != 0) and re.match('^}', l):
+      vEnd = i
+      break
+
+  for i in range(int(vBegin) + 1,int(vEnd) + 1):
+    # match: four spaces, the vlan 'name', one space, curly brace
+    if re.match('^    [\S]+ {', hostConfig[i]):
+      name=re.sub('{','',hostConfig[i]).strip()
+      j = 1
+      while True:
+        if 'vlan-id' in hostConfig[i + j]:
+          vlan=re.sub('vlan-id', '', hostConfig[i + j]).strip()
+          vlan=re.sub(';', '', vlan)
+          break
+        else:
+          j += 1
+      vlanDict[vlan] = name
+
+  vlanList=vlanDict.items()
+  return(vlanList)
+
+def look_in_juniper_interfaces(hostName, hostConfig):
+  # This is different from the 'vlans' section, as we have to keep track
+  # of the interfaces too. Also, the same vlan-id may exist under different
+  # names on different interfaces. For this reason the 'vlanList' data
+  # structure in this def: is a list of lists
+  vlanList=[]
+  iBegin = 0
+  for i, l in enumerate(hostConfig):
+    if re.match('^interfaces {', l):
+      iBegin = i
+    if (iBegin != 0) and re.match('^}', l):
+      iEnd = i
+      break
+
+  for i in range(int(iBegin) + 1, int(iEnd) +1):
+    if re.match("^    [\S]+[\s]*[\S]+ {", hostConfig[i]):
+      iFace=re.sub('{','',hostConfig[i]).strip()
+      inUnit = 0
+      name = ''
+      while ( not (re.match('^    }',hostConfig[i]))):
+        i += 1
+        if re.match("^        .*unit", hostConfig[i]):
+          unit=re.sub("^        .*unit ",'', hostConfig[i]).strip()
+          unit=re.sub('{', '', unit).strip()
+          inUnit = 1
+        if (re.match('^        }', hostConfig[i])) and (inUnit == 1):
+          inUnit = 0
+
+        if (inUnit ==1):
+          if "description" in hostConfig[i]:
+            name=re.sub('[\s]+description','',hostConfig[i]).strip()
+            name=re.sub('[";]','',name)
+          if "vlan-id" in hostConfig[i]:
+            vlan=re.sub('[\s]+vlan-id', '', hostConfig[i]).strip()
+            vlan=re.sub(';', '', vlan)
+
+            if "-range" in vlan:
+              vBegin = re.sub('^-range ','', vlan)
+              vBegin = re.sub('-[0-9]+$', '', vBegin)
+              vEnd   = re.sub('-range [0-9]+-', '', vlan)
+              for v in range(int(vBegin), int(vEnd) +1):
+                vlanList.append([v,name,iFace])
+            else:
+              vlanList.append([vlan,name,iFace])
+  return(vlanList)
+
+def write_list_to_file(outFile, outList):
+  f = open(outFile, "w")
+  for l in outList:
+    f.write(l)
+  f.close
+
+def check_out_vs_dst(hostName, vlanList):
+  outFile = outputDir + "/" + hostName
+  outList = []
+  for l in sorted(vlanList, key=lambda x: int(x[0])):
+    o=''
+    for i, sl in enumerate(l):
+      if o == '':
+        o = str(sl)
+      else:
+        o = o + ";" + str(sl)
+    # we want the resulting 'csv' to have the same number of fields, whether
+    # or not the source config has 'interface' for the vlan:
+    if i == 1:
+      o = o + ";"
+    o = o + "\n"
+    outList.append(o)
+  if os.path.isfile(outFile):
+    dstList = open(outFile, 'r').readlines()
+    compare = set(sorted(outList)) & set(sorted(dstList))
+    if (len(compare) != len(dstList)) or (len(compare) != len(outList)):
+      write_list_to_file(outFile, outList)
+  else:
+    write_list_to_file(outFile, outList)
+
+
+def is_juniper(hostName):
+# print "juniper:  ", host
+  vlanList=dict()
+  inFile = configsDir + "/" + hostName
+  outFile = outputDir + "/" + hostName
+  f = open(inFile)
+  hostConfig=f.readlines()
+  f.close()
+
+  # find device model
+  for l in hostConfig:
+    if re.match('^# Chassis', l):
+      l = re.sub('[\s]+', ':', l)
+      ll = l.split(':')
+      model = ll[3]
+
+  if "EX" in model:
+    vlanList=look_in_juniper_vlans_section(hostName, hostConfig)
+    check_out_vs_dst(hostName,vlanList)
+  elif "MX" in model:
+    vlanList=look_in_juniper_interfaces(hostName, hostConfig)
+    check_out_vs_dst(hostName, vlanList)
+  elif "SRX" in model:
+    vlanListInterfaces=look_in_juniper_interfaces(hostName, hostConfig)
+    vlanListVlanSection=look_in_juniper_vlans_section(hostName, hostConfig)
+    vlanList=vlanListInterfaces + vlanListVlanSection
+    check_out_vs_dst(hostName, vlanList)
+  elif "Virtual" in model:
+    vlanList=look_in_juniper_vlans_section(hostName, hostConfig)
+    check_out_vs_dst(hostName, vlanList)
+  else:
+    print "model not recognized, skipping"
+
+
+try:
+  os.makedirs(outputDir)
+except OSError as e:
+  if e.errno != errno.EEXIST:
+    raise
+
+for host in hostList:
+  if host in routerDB:
+    if routerDB[host] == 'arista':
+      is_arista(host)
+    elif routerDB[host] == 'juniper':
+      is_juniper(host)
+    else:
+      print "unknown:  ", host
+