From 96c9118077a4ecd193f3e32dea3308ce7c95cdb5 Mon Sep 17 00:00:00 2001 From: Johannes Garm Nielsen Date: Tue, 13 Nov 2018 12:28:56 +0100 Subject: Reworked scripts, added testing material, added README.md --- README.md | 34 +++++++++++ pythonDocUpload.yml | 4 ++ testing/docker-compose.yml | 17 ++++++ testing/pythonDocUpload.yml | 4 ++ uploadDoc.py | 133 -------------------------------------------- uploadDocs.py | 133 ++++++++++++++++++++++++++++++++++++++++++++ wipeDocs.py | 19 +++---- 7 files changed, 201 insertions(+), 143 deletions(-) create mode 100644 README.md create mode 100644 pythonDocUpload.yml create mode 100644 testing/docker-compose.yml create mode 100644 testing/pythonDocUpload.yml delete mode 100644 uploadDoc.py create mode 100644 uploadDocs.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..b20f864 --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +# Python Doc Upload - Setup, Usage and Testing: + + +## SETUP: + +* The doc uploader needs access to a checkout of the ndn-sysconf git repository, puppet (for puppet strings) and the ability to connect to portal.nordu.net. +* Install python libraries: + * pip install atlassian-python-api pyyaml json2html pypandoc +* Edit pythonDocUpload.yml - the values should be self explanatory. +* Done! + +## USAGE: + +There are two scripts: uploadDocs.py and wipeDocs.py. + +Run uploadDocs to generate and upload the newest docs. Example: + +```python3 uploadDocs.py path/to/ndn-sysconf-checkout``` + +The script takes a minute to run. It should finish with a list of the errors it encountered while generating documentation. + +If documentation is removed from ndn-sysconf, uploadDocs will not remove their old pages from confluence. If these need to be pruned, I suggest first running wipeDocs to remove the entire space Example: + +```python3 wipeDocs.py force``` + +Doing so will obvious ruin the historics but expanding the uploadDocs script to prune intelligently seems like a poor time investment at the time of writing. + +## Testing: + +In the testing folder, you will find a docker-compose file which sets up an environment with two containers: One running Confluence and one running Puppet. The confluence instance needs to be set up with a trial version before testing can begin and the puppet docker needs an ndn-sysconf checkout available in a volume mount (as it is in the docker-compose.yml). Once this is set up, you can run the scripts from the puppet docker and observe the changes on the confluence docker. + +## Development notes: + +Confluence page names are globally unique, hence the naming scheme for the manifests pages and others. The exact processing of the documents done in uploadDocs may be overkill but at the time of writing it is unclear whether these scripts will be needed and it does work. \ No newline at end of file diff --git a/pythonDocUpload.yml b/pythonDocUpload.yml new file mode 100644 index 0000000..5567cad --- /dev/null +++ b/pythonDocUpload.yml @@ -0,0 +1,4 @@ +confluenceUrl: "http://confluence:8090" +confluenceUser: apiuser +confluencePass: apiuserpass +spaceName: TES diff --git a/testing/docker-compose.yml b/testing/docker-compose.yml new file mode 100644 index 0000000..04a819f --- /dev/null +++ b/testing/docker-compose.yml @@ -0,0 +1,17 @@ +version: '3.3' +services: + puppetserver: + image: puppet/puppetserver-standalone:latest + volumes: + - ./ndn-sysconf:/opt/data/etc + networks: + - test + confluence: + image: atlassian/confluence-server:latest + networks: + - test + ports: + - "8090:8090" + - "8091:8091" +networks: + test: diff --git a/testing/pythonDocUpload.yml b/testing/pythonDocUpload.yml new file mode 100644 index 0000000..5567cad --- /dev/null +++ b/testing/pythonDocUpload.yml @@ -0,0 +1,4 @@ +confluenceUrl: "http://confluence:8090" +confluenceUser: apiuser +confluencePass: apiuserpass +spaceName: TES diff --git a/uploadDoc.py b/uploadDoc.py deleted file mode 100644 index a9a384e..0000000 --- a/uploadDoc.py +++ /dev/null @@ -1,133 +0,0 @@ -import subprocess, fileinput, pprint, re, os, sys, json, pypandoc, yaml -from atlassian import Confluence -from json2html import * - -def prepareMarkdown(sourcePath): - fileContent = pypandoc.convert_file(sourcePath, "html") - - # Alter document to confluence standard so we can tell if the page needs updating - # Strip trailing whitespace - fileContent = fileContent.strip() - - # Remove ids since confluence file comparison fails with them - fileContent = re.sub(r" id=\"[\w-]*\"", "", fileContent) - - # Replace new lines with space - fileContent = re.sub(r"\n", " ", fileContent) - - # Remove markdown comments - fileContent = re.sub(r"", "", fileContent) - - # Custom character handling - fileContent = fileContent.replace('ó', u'ó') - fileContent = fileContent.replace('’', '’') - - return fileContent - -def prepareJSON(jsonFileContent): - jsonDict = json.loads(jsonFileContent) - htmlTable = json2html.convert(jsonDict) - htmlTable = htmlTable.strip() - - # Remove ids since confluence file comparison fails with them - htmlTable = re.sub(r" id=\"[\w-]*\"", "", htmlTable) - - # Replace new lines with space - htmlTable = re.sub(r"\n", " ", htmlTable) - - # Remove markdown comments - htmlTable = re.sub(r"", "", htmlTable) - - # Custom character handling - htmlTable = htmlTable.replace('ó', u'ó') - htmlTable = htmlTable.replace('’', '’') - - return htmlTable - -pp = pprint.PrettyPrinter(indent=2) - -configuration = yaml.load(open("docUploadConfig.yml", "r").read()) -print(configuration) - -confluence = Confluence( - url=configuration.get('confluenceUrl'), - username=configuration.get('confluenceUser'), - password=configuration.get('confluencePass')) - -hostsDir = sys.argv[1] + "/hosts" -puppetDir = sys.argv[1] + "/puppet/modules/ndn" -docsDir = sys.argv[1] + "/docs" - -errors = "" - -# Create hosts page -hostsPageId = confluence.get_page_id(configuration.get('spaceName'), "hosts") -if hostsPageId == None: - print("Creating hosts page") - hostsPageId = confluence.create_page(configuration.get('spaceName'), "hosts", '').get("id") - -# Create manifests pages -for dirname in os.listdir(hostsDir): - print(dirname) - # Create individual host page and manifests - dirPath = hostsDir + "/" + dirname - landingContent = "" - if os.path.isfile(dirPath + "/manifest.json"): - landingContent += prepareJSON(open(dirPath + "/manifest.json", "r").read()) - if os.path.isfile(dirPath + "/README.md"): - landingContent += prepareMarkdown(dirPath + "/README.md") - - dirPageReturn = confluence.update_or_create(hostsPageId, dirname, landingContent) - dirPageId = confluence.get_page_id(configuration.get('spaceName'), dirname) - manifestsPageReturn = confluence.update_or_create(dirPageId, "manifests - " + dirname, 'placeholder') - - for filename in os.listdir(dirPath + "/manifests"): - if filename.endswith(".pp"): - try: - subprocess.run(["puppet", "strings", "generate", "--format", "markdown", dirPath + "/" + filename, "--out", "temp.md"]) - fileContent = prepareMarkdown("temp.md") - pageReturn = confluence.update_or_create(manifestsPageReturn.get("id"), filename + " - " + dirname, fileContent) - except Exception as e: - print("Error processing manifest file: " + dirPath + "/" + filename) - errors += dirPath + "/manifests/" + filename + " MESSAGE: " + str(e) + "\n" - - -dirPageId = confluence.get_page_id(configuration.get('spaceName'), "ndn") -if dirPageId == None: - dirPageId = confluence.create_page(configuration.get('spaceName'), "ndn", '').get("id") - -modulesPageReturn = confluence.update_or_create(dirPageId, "puppet/modules/ndn", "") - -manifestsDir = puppetDir + "/manifests" - -if os.path.isdir(manifestsDir): - for manifestFile in os.listdir(manifestsDir): - try: - if manifestFile.endswith(".pp"): - subprocess.run(["puppet", "strings", "generate", "--format", "markdown", manifestsDir + "/" + manifestFile, "--out", "temp.md"]) - fileContent = prepareMarkdown("temp.md") - pp.pprint("page: " + manifestFile + " - " + dirname) - pageReturn = confluence.update_or_create(modulesPageReturn.get("id"), manifestFile + " - " + dirname, fileContent) - except Exception as e: - print("Error processing puppet manifest: " + dirname + "/" + manifestFile) - errors += manifestsDir + "/" + manifestFile + " MESSAGE: " + str(e) + "\n" - -docsPageId = confluence.get_page_id(configuration.get('spaceName'), "docs") -if docsPageId == None: - print("Creating docs page") - docsPageId = confluence.create_page(configuration.get('spaceName'), "docs", '').get("id") - -for docsName in os.listdir(docsDir): - try: - fileContent = prepareMarkdown(docsDir + "/" + docsName) - pp.pprint("page: " + docsDir + " - " + docsName) - pageReturn = confluence.update_or_create(docsPageId, docsName + " - docs", fileContent) - except Exception as e: - print("Error processing docs file: " + docsDir + "/" + docsName) - errors += docsDir + "/" + docsName + " MESSAGE: " + str(e) + "\n" - -if errors != "": - print("Errors encountered while handling following docs: ") - print(errors) - -print("------- DONE -------") \ No newline at end of file diff --git a/uploadDocs.py b/uploadDocs.py new file mode 100644 index 0000000..8d2d02c --- /dev/null +++ b/uploadDocs.py @@ -0,0 +1,133 @@ +import subprocess, fileinput, pprint, re, os, sys, json, pypandoc, yaml +from atlassian import Confluence +from json2html import * + +def prepareMarkdown(sourcePath): + fileContent = pypandoc.convert_file(sourcePath, "html") + + # Alter document to confluence standard so we can tell if the page needs updating + # Strip trailing whitespace + fileContent = fileContent.strip() + + # Remove ids since confluence file comparison fails with them + fileContent = re.sub(r" id=\"[\w-]*\"", "", fileContent) + + # Replace new lines with space + fileContent = re.sub(r"\n", " ", fileContent) + + # Remove markdown comments + fileContent = re.sub(r"", "", fileContent) + + # Custom character handling + fileContent = fileContent.replace('ó', u'ó') + fileContent = fileContent.replace('’', '’') + + return fileContent + +def prepareJSON(jsonFileContent): + jsonDict = json.loads(jsonFileContent) + htmlTable = json2html.convert(jsonDict) + htmlTable = htmlTable.strip() + + # Remove ids since confluence file comparison fails with them + htmlTable = re.sub(r" id=\"[\w-]*\"", "", htmlTable) + + # Replace new lines with space + htmlTable = re.sub(r"\n", " ", htmlTable) + + # Remove markdown comments + htmlTable = re.sub(r"", "", htmlTable) + + # Custom character handling + htmlTable = htmlTable.replace('ó', u'ó') + htmlTable = htmlTable.replace('’', '’') + + return htmlTable + +pp = pprint.PrettyPrinter(indent=2) + +configuration = yaml.load(open("pythonDocUpload.yml", "r").read()) +print(configuration) + +confluence = Confluence( + url=configuration.get('confluenceUrl'), + username=configuration.get('confluenceUser'), + password=configuration.get('confluencePass')) + +hostsDir = sys.argv[1] + "/hosts" +puppetDir = sys.argv[1] + "/puppet/modules/ndn" +docsDir = sys.argv[1] + "/docs" + +errors = "" + +# Create hosts page +hostsPageId = confluence.get_page_id(configuration.get('spaceName'), "hosts") +if hostsPageId == None: + print("Creating hosts page") + hostsPageId = confluence.create_page(configuration.get('spaceName'), "hosts", '').get("id") + +# Create manifests pages +for dirname in os.listdir(hostsDir): + print(dirname) + # Create individual host page and manifests + dirPath = hostsDir + "/" + dirname + landingContent = "" + if os.path.isfile(dirPath + "/manifest.json"): + landingContent += prepareJSON(open(dirPath + "/manifest.json", "r").read()) + if os.path.isfile(dirPath + "/README.md"): + landingContent += prepareMarkdown(dirPath + "/README.md") + + dirPageReturn = confluence.update_or_create(hostsPageId, dirname, landingContent) + dirPageId = confluence.get_page_id(configuration.get('spaceName'), dirname) + manifestsPageReturn = confluence.update_or_create(dirPageId, "manifests - " + dirname, 'placeholder') + + for filename in os.listdir(dirPath + "/manifests"): + if filename.endswith(".pp"): + try: + subprocess.run(["puppet", "strings", "generate", "--format", "markdown", dirPath + "/" + filename, "--out", "temp.md"]) + fileContent = prepareMarkdown("temp.md") + pageReturn = confluence.update_or_create(manifestsPageReturn.get("id"), filename + " - " + dirname, fileContent) + except Exception as e: + print("Error processing manifest file: " + dirPath + "/" + filename) + errors += dirPath + "/manifests/" + filename + " MESSAGE: " + str(e) + "\n" + + +dirPageId = confluence.get_page_id(configuration.get('spaceName'), "ndn") +if dirPageId == None: + dirPageId = confluence.create_page(configuration.get('spaceName'), "ndn", '').get("id") + +modulesPageReturn = confluence.update_or_create(dirPageId, "puppet/modules/ndn", "") + +manifestsDir = puppetDir + "/manifests" + +if os.path.isdir(manifestsDir): + for manifestFile in os.listdir(manifestsDir): + try: + if manifestFile.endswith(".pp"): + subprocess.run(["puppet", "strings", "generate", "--format", "markdown", manifestsDir + "/" + manifestFile, "--out", "temp.md"]) + fileContent = prepareMarkdown("temp.md") + pp.pprint("page: " + manifestFile + " - " + dirname) + pageReturn = confluence.update_or_create(modulesPageReturn.get("id"), manifestFile + " - " + dirname, fileContent) + except Exception as e: + print("Error processing puppet manifest: " + dirname + "/" + manifestFile) + errors += manifestsDir + "/" + manifestFile + " MESSAGE: " + str(e) + "\n" + +docsPageId = confluence.get_page_id(configuration.get('spaceName'), "docs") +if docsPageId == None: + print("Creating docs page") + docsPageId = confluence.create_page(configuration.get('spaceName'), "docs", '').get("id") + +for docsName in os.listdir(docsDir): + try: + fileContent = prepareMarkdown(docsDir + "/" + docsName) + pp.pprint("page: " + docsDir + " - " + docsName) + pageReturn = confluence.update_or_create(docsPageId, docsName + " - docs", fileContent) + except Exception as e: + print("Error processing docs file: " + docsDir + "/" + docsName) + errors += docsDir + "/" + docsName + " MESSAGE: " + str(e) + "\n" + +if errors != "": + print("Errors encountered while handling following docs: ") + print(errors) + +print("------- DONE -------") \ No newline at end of file diff --git a/wipeDocs.py b/wipeDocs.py index 390f167..11e04de 100644 --- a/wipeDocs.py +++ b/wipeDocs.py @@ -1,19 +1,18 @@ -import pprint, json +import pprint, json, yaml, sys from atlassian import Confluence -from base64 import b64encode pp = pprint.PrettyPrinter(indent=2) -confluence = Confluence( - url='http://localhost:8090', - username='apiuser', - password='apiuserpass') - -spaceName = sys.argv[1] +configuration = yaml.load(open("pythonDocUpload.yml", "r").read()) -pages = confluence.get_all_pages_from_space(spaceName) +confluence = Confluence( + url=configuration.get('confluenceUrl'), + username=configuration.get('confluenceUser'), + password=configuration.get('confluencePass')) -# jsonPages = json.loads(pages) +pages = confluence.get_all_pages_from_space(configuration.get('spaceName')) +if (len(sys.argv) > 0 && sys.argv[1] != "force"): + input("You are about to wipe everything in the space " + configuration.get('spaceName') + " - press Enter to continue.") for page in pages: print(pp.pprint(page)) -- cgit v1.1