initial commit, 'seems to work', README, notes

3 files changed, 279 insertions, 0 deletions

diff --git a/README b/README
new file mode 100644
index 0000000..7d4068a
--- /dev/null
+++ b/README
@@ -0,0 +1,33 @@
+This is a relatively simple piece of code, to solve a relatively central 
+lack of knowledge:
+
+Which VLANs exist on which devices?
+
+The script is run at some interval from CRON, parses the config files
+gathered by RANCID, and extracts VLAN info
+
+The results are stored locally in 'vlanscrape's $HOME/data/ as one single
+'csv' file per device. Later, that directory will be added into this repo
+and updates will be pushed here.
+
+The format of the output files is:
+<vlan-id>;<vlan-name/description>;<interface-info>
+
+-where:
+vlan-id    = integer
+vlan-name  = name or 'description' (if one exists)
+iface-info = interface the vlan exists on (mostly with routers)
+
+Semicolon is chosen as delimiter, since the character is used in a similar
+role in junos configs, and the risk of 'collision' should be reduced as a
+consequence.
+
+Because this script relies on reading the local RANCID files, it needs to run
+where RANCID runs. Currently that means 'statler'.
+
+The script uses 'core' modules or custom written functions exclusively. This
+is to make it as "portable" as possible. The only 'requirement' is that the
+python version should be higher than 2.7, but lower than 3.0
+
+There is definitely room for improvement, but the script does what it says
+on the tin - at least as long as nothing unexpected happens...
diff --git a/code/vlanscrape b/code/vlanscrape
new file mode 100755
index 0000000..25a05da
--- /dev/null
+++ b/code/vlanscrape
@@ -0,0 +1,226 @@
+#!/usr/bin/python
+#
+# 'router', 'host' and 'device' are used interchangably in this script
+
+configsDir   = "/store/rancid/nordunet/configs"
+outputDir    = "/home/vlanscrape/data"
+routerDBFile = "/store/rancid/nordunet/router.db"
+routerDB     = dict()
+
+# populate 'routerDB' with the contents of routerDBFile
+f = open(routerDBFile, 'r')
+for line in f:
+  hostname, vendor, state = line.split(";")
+  # ignore 'state' as it is of no consequence for the purpose of this script
+  routerDB[hostname]=vendor
+f.close()
+
+import os
+import errno
+import re
+import shutil
+import time
+
+# populate 'hostList'
+hostList = os.listdir(configsDir)
+if 'CVS' in hostList:
+  hostList.remove('CVS')
+if '.cvsignore' in hostList:
+  hostList.remove('.cvsignore')
+
+def is_arista(hostName):
+  vlanDict=dict()
+# print "arista:   ", hostName
+  inFile = configsDir + "/" + hostName
+  outFile = outputDir + "/" + hostName
+  f = open(inFile)
+  hostConfig=f.readlines()
+  f.close()
+  for i, l in enumerate(hostConfig):
+    if re.match("^vlan", hostConfig[i]):
+      # The config allows a comma separated list of multiple vlan ids,
+      # with the same name, and it also allows vlans to be unnamed.
+      # The simplest way to handle this is to grab the (lack of) name first
+      # and apply it to the (list of) vlan(s) after.
+      j = 1
+      while True:
+        if re.match(".*name ", hostConfig[i + j]):
+          name = re.sub(".*name ", "", hostConfig[i + j].rstrip())
+          break
+        elif re.match('^!', hostConfig[i + j]):
+          name = ""
+          break
+        else:
+          j += 1
+
+      vlan = re.sub(".*vlan ", "", hostConfig[i].rstrip())
+      if "," in vlan:
+        for v in vlan.split(","):
+          vlanDict[v] = name
+      elif "-" in vlan:
+        first, last = vlan.split("-")
+        for v in range(int(first),(int(last) + 1)):
+          vlanDict[v] = name
+      else:
+        vlanDict[vlan] = name
+
+  vlanList=vlanDict.items()
+  check_out_vs_dst(hostName,vlanList)
+
+def look_in_juniper_vlans_section(hostName, hostConfig):
+  # Since vlans in the 'vlans' section (of a switch config) can only exist
+  # once per 'device', using a dict here makes sense, as it ensures there will
+  # only be one instance.
+  vlanDict=dict()
+  vBegin = 0
+  vEnd = 0
+  for i, l in enumerate(hostConfig):
+    if re.match('^vlans {', l):
+      vBegin = i
+    if (vBegin != 0) and re.match('^}', l):
+      vEnd = i
+      break
+
+  for i in range(int(vBegin) + 1,int(vEnd) + 1):
+    # match: four spaces, the vlan 'name', one space, curly brace
+    if re.match('^    [\S]+ {', hostConfig[i]):
+      name=re.sub('{','',hostConfig[i]).strip()
+      j = 1
+      while True:
+        if 'vlan-id' in hostConfig[i + j]:
+          vlan=re.sub('vlan-id', '', hostConfig[i + j]).strip()
+          vlan=re.sub(';', '', vlan)
+          break
+        else:
+          j += 1
+      vlanDict[vlan] = name
+
+  vlanList=vlanDict.items()
+  return(vlanList)
+
+def look_in_juniper_interfaces(hostName, hostConfig):
+  # This is different from the 'vlans' section, as we have to keep track
+  # of the interfaces too. Also, the same vlan-id may exist under different
+  # names on different interfaces. For this reason the 'vlanList' data
+  # structure in this def: is a list of lists
+  vlanList=[]
+  iBegin = 0
+  for i, l in enumerate(hostConfig):
+    if re.match('^interfaces {', l):
+      iBegin = i
+    if (iBegin != 0) and re.match('^}', l):
+      iEnd = i
+      break
+
+  for i in range(int(iBegin) + 1, int(iEnd) +1):
+    if re.match("^    [\S]+[\s]*[\S]+ {", hostConfig[i]):
+      iFace=re.sub('{','',hostConfig[i]).strip()
+      inUnit = 0
+      name = ''
+      while ( not (re.match('^    }',hostConfig[i]))):
+        i += 1
+        if re.match("^        .*unit", hostConfig[i]):
+          unit=re.sub("^        .*unit ",'', hostConfig[i]).strip()
+          unit=re.sub('{', '', unit).strip()
+          inUnit = 1
+        if (re.match('^        }', hostConfig[i])) and (inUnit == 1):
+          inUnit = 0
+
+        if (inUnit ==1):
+          if "description" in hostConfig[i]:
+            name=re.sub('[\s]+description','',hostConfig[i]).strip()
+            name=re.sub('[";]','',name)
+          if "vlan-id" in hostConfig[i]:
+            vlan=re.sub('[\s]+vlan-id', '', hostConfig[i]).strip()
+            vlan=re.sub(';', '', vlan)
+
+            if "-range" in vlan:
+              vBegin = re.sub('^-range ','', vlan)
+              vBegin = re.sub('-[0-9]+$', '', vBegin)
+              vEnd   = re.sub('-range [0-9]+-', '', vlan)
+              for v in range(int(vBegin), int(vEnd) +1):
+                vlanList.append([v,name,iFace])
+            else:
+              vlanList.append([vlan,name,iFace])
+  return(vlanList)
+
+def write_list_to_file(outFile, outList):
+  f = open(outFile, "w")
+  for l in outList:
+    f.write(l)
+  f.close
+
+def check_out_vs_dst(hostName, vlanList):
+  outFile = outputDir + "/" + hostName
+  outList = []
+  for l in sorted(vlanList, key=lambda x: int(x[0])):
+    o=''
+    for i, sl in enumerate(l):
+      if o == '':
+        o = str(sl)
+      else:
+        o = o + ";" + str(sl)
+    # we want the resulting 'csv' to have the same number of fields, whether
+    # or not the source config has 'interface' for the vlan:
+    if i == 1:
+      o = o + ";"
+    o = o + "\n"
+    outList.append(o)
+  if os.path.isfile(outFile):
+    dstList = open(outFile, 'r').readlines()
+    compare = set(sorted(outList)) & set(sorted(dstList))
+    if (len(compare) != len(dstList)) or (len(compare) != len(outList)):
+      write_list_to_file(outFile, outList)
+  else:
+    write_list_to_file(outFile, outList)
+
+
+def is_juniper(hostName):
+# print "juniper:  ", host
+  vlanList=dict()
+  inFile = configsDir + "/" + hostName
+  outFile = outputDir + "/" + hostName
+  f = open(inFile)
+  hostConfig=f.readlines()
+  f.close()
+
+  # find device model
+  for l in hostConfig:
+    if re.match('^# Chassis', l):
+      l = re.sub('[\s]+', ':', l)
+      ll = l.split(':')
+      model = ll[3]
+
+  if "EX" in model:
+    vlanList=look_in_juniper_vlans_section(hostName, hostConfig)
+    check_out_vs_dst(hostName,vlanList)
+  elif "MX" in model:
+    vlanList=look_in_juniper_interfaces(hostName, hostConfig)
+    check_out_vs_dst(hostName, vlanList)
+  elif "SRX" in model:
+    vlanListInterfaces=look_in_juniper_interfaces(hostName, hostConfig)
+    vlanListVlanSection=look_in_juniper_vlans_section(hostName, hostConfig)
+    vlanList=vlanListInterfaces + vlanListVlanSection
+    check_out_vs_dst(hostName, vlanList)
+  elif "Virtual" in model:
+    vlanList=look_in_juniper_vlans_section(hostName, hostConfig)
+    check_out_vs_dst(hostName, vlanList)
+  else:
+    print "model not recognized, skipping"
+
+
+try:
+  os.makedirs(outputDir)
+except OSError as e:
+  if e.errno != errno.EEXIST:
+    raise
+
+for host in hostList:
+  if host in routerDB:
+    if routerDB[host] == 'arista':
+      is_arista(host)
+    elif routerDB[host] == 'juniper':
+      is_juniper(host)
+    else:
+      print "unknown:  ", host
+
diff --git a/meta/info b/meta/info
new file mode 100644
index 0000000..8bb22ab
--- /dev/null
+++ b/meta/info
@@ -0,0 +1,20 @@
+User created with:
+
+-bash-4.4$ sudo useradd -m -G rancid -c "user to scrape vlan info from RANCID" vlanscrape
+
+---
+Script 'installed' by:
+
+scp to statler
+-bash-4.4$ sudo chown root:wheel vlanscrape
+-bash-4.4$ sudo mv vlanscrape /usr/local/bin/
+
+`python` lives in /usr/pkg/bin/ on netbsd, so the 'bang' needs to be modified
+accordingly
+
+---
+Cron entry:
+
+-bash-4.4$ sudo crontab -l -u vlanscrape
+6 * * * * /usr/local/bin/vlanscrape
+