mstarch: updating documentation generation to adjust for github

Fw/Python/docs/gendoc.bash

@@ -2,17 +2,5 @@
 
 # A quick file for generating docs
 DIRECTORY="../../../docs/UsersGuide/api/python/fprime/html"
-rm -r "${DIRECTORY}"
 mkdir -p "${DIRECTORY}"
-
 sphinx-build . "${DIRECTORY}"
-(
-    cd "${DIRECTORY}"
-    for reference in _*
-    do
-        newref="${reference#_}"
-        mv "${reference}" "${newref}"
-        sed -i.bakobak 's/'${reference}'/'${newref}'/g' `find . -type f -exec grep -Iq . {} \; -print`
-    done
-    find . -name "*.bakobak" -delete
-)

Gds/docs/gendoc.bash

@@ -2,17 +2,6 @@
 
 # A quick file for generating docs
 DIRECTORY="../../docs/UsersGuide/api/python/fprime-gds/html"
-rm -r "${DIRECTORY}"
 mkdir -p "${DIRECTORY}"
 
 sphinx-build . "${DIRECTORY}"
-(
-    cd "${DIRECTORY}"
-    for reference in _*
-    do
-        newref="${reference#_}"
-        mv "${reference}" "${newref}"
-        sed -i.bakobak 's/'${reference}'/'${newref}'/g' `find . -type f -exec grep -Iq . {} \; -print`
-    done
-    find . -name "*.bakobak" -delete
-)

docs/_config.yml

@@ -1,3 +1,4 @@
 title: "F´"
 description: "Flight Software & Embedded Systems Framework"
-theme: jekyll-theme-slate
+theme: jekyll-theme-slate,
+include: UsersGuide/api/

docs/doxygen/Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = "F´ Flight Software - C/C++ Documentation"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = NASA-v1.5.0
+PROJECT_NUMBER         = NASA-v1.6.0
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -51,7 +51,7 @@ PROJECT_BRIEF          = "A framework for building embedded system applications
 # pixels and the maximum width should not exceed 200 pixels. Doxygen will copy
 # the logo to the output directory.
 
-PROJECT_LOGO           =
+PROJECT_LOGO           = ./docs/doxygen/logo.png
 
 # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
 # into which the generated documentation will be written. If a relative path is
@@ -170,7 +170,7 @@ FULL_PATH_NAMES        = YES
 # will be relative from the directory where doxygen is started.
 # This tag requires that the tag FULL_PATH_NAMES is set to YES.
 
-STRIP_FROM_PATH        = ./ ./Ref/build-fprime-automatic-default/ ./Ref/build-fprime-automatic-default/F-Prime/
+STRIP_FROM_PATH        = ./ ./docs-build-for-docs/ ./docs-build-for-docs/F-Prime/ ./Ref/build-fprime-automatic-default/ ./Ref/build-fprime-automatic-default/F-Prime/
 
 # The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the
 # path mentioned in the documentation of a class, which tells the reader which
@@ -179,7 +179,7 @@ STRIP_FROM_PATH        = ./ ./Ref/build-fprime-automatic-default/ ./Ref/build-fp
 # specify the list of include paths that are normally passed to the compiler
 # using the -I flag.
 
-STRIP_FROM_INC_PATH    = ./ ./Ref/build-fprime-automatic-default/ ./Ref/build-fprime-automatic-default/F-Prime/
+STRIP_FROM_INC_PATH    = ./ ./docs-build-for-docs/ ./docs-build-for-docs/F-Prime/ ./Ref/build-fprime-automatic-default/ ./Ref/build-fprime-automatic-default/F-Prime/
 
 # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but
 # less readable) file names. This can be useful is your file systems doesn't
@@ -915,7 +915,7 @@ EXCLUDE_SYMLINKS       = NO
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories for example use the pattern */test/*
 
-EXCLUDE_PATTERNS       = */test/* */GTest/* */build-clion/* */build*ut/* */Autocoders/MagicDrawCompPlugin/* */cmake/* 
+EXCLUDE_PATTERNS       = */test/* */GTest/* */build-*/* */Autocoders/MagicDrawCompPlugin/* */cmake/* 
 
 # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
 # (namespaces, classes, functions, etc.) that should be excluded from the

docs/doxygen/generate_docs.bash

@@ -4,13 +4,20 @@
 ####
 SOURCE_DIR=`dirname $BASH_SOURCE`
 
-DOXYGEN="doxygen"
 FPRIME=`cd ${SOURCE_DIR}/../../; pwd`
+APIDOCS="${FPRIME}/docs/UsersGuide/api"
 
 DOXY_OUTPUT="${FPRIME}/docs/UsersGuide/api/c++"
 CMAKE_OUTPUT="${FPRIME}/docs/UsersGuide/api/cmake"
 PY_OUTPUT="${FPRIME}/docs/UsersGuide/api/python"
 
+DOXYGEN="${1:-$(which doxygen)}"
+if [ ! -x "${DOXYGEN}" ]
+then
+    echo "[ERROR] Could not find doxygen, please supply it as first argument"
+    exit 221
+fi
+
 FPRIME_UTIL=`which fprime-util`
 if [[ "${FPRIME_UTIL}" == "" ]]
 then
@@ -18,48 +25,63 @@ then
     exit 1
 fi
 
+function github_page_adjustment
+{
+    DIRECTORY="${1}"
+    shift
+    echo "[INFO] Munching '_'s in ${DIRECTORY}" 
+    python "${SOURCE_DIR}/gh_pages.py" "${DIRECTORY}"
+}
+function clobber
+{
+    DIRECTORY="${1}"
+    shift
+    if [[ "${DIRECTORY}" != ${APIDOCS}* ]]
+    then
+        echo "[ERROR] Cannot clobber ${DIRECTORY} as it is not a child of ${APIDOCS}"
+        exit 233
+    fi 
+    read -p "[ENTER] Clobbering ${DIRECTORY}. CTRL-C to abort." tmp
+    rm -r "${DIRECTORY}"
+}
+
 # Doxygen generation
 (
     cd "${FPRIME}"
-    if [ -e "${DOXY_OUTPUT}.prev" ]
-    then
-        echo "[ERROR] Backup already exists at ${DOXY_OUTPUT}.prev"
-        exit 1
-    fi
-    fprime-util build -j32
+    clobber "${DOXY_OUTPUT}"
+    (
+        mkdir -p "${FPRIME}/docs-build-for-docs"
+        cd "${FPRIME}/docs-build-for-docs"
+        cmake "${FPRIME}" -DCMAKE_BUILD_TYPE=RELEASE
+    )
+    fprime-util build -b "${FPRIME}/docs-build-for-docs" -j32
     if (( $? != 0 ))
     then
         echo "[ERROR] Failed to build fprime please generate build cache"
         exit 2 
     fi
-    mv "${DOXY_OUTPUT}" "${DOXY_OUTPUT}.prev"
     ${DOXYGEN} "${FPRIME}/docs/doxygen/Doxyfile"
+    rm -r "${FPRIME}/docs-build-for-docs"
 ) || exit 1
 
 # CMake
 (
     cd "${FPRIME}"
-    if [ -e "${CMAKE_OUTPUT}.prev" ]
-    then
-        echo "[ERROR] Backup already exists at ${CMAKE_OUTPUT}.prev"
-        exit 1
-    fi
-    mv "${CMAKE_OUTPUT}" "${CMAKE_OUTPUT}.prev"
+    clobber "${CMAKE_OUTPUT}"
     mkdir -p "${CMAKE_OUTPUT}"
     "${FPRIME}/cmake/docs/docs.py" "${FPRIME}/cmake/" "${FPRIME}/docs/UsersGuide/api/cmake"
 ) || exit 1
 
 # Python
 (
+    clobber "${PY_OUTPUT}"
     cd "${FPRIME}/Fw/Python/docs"
-    if [ -e "${PY_OUTPUT}.prev" ]
-    then
-        echo "[ERROR] Backup already exists at ${PY_OUTPUT}.prev"
-        exit 1
-    fi
-    mv "${PY_OUTPUT}" "${PY_OUTPUT}.prev"
     "${FPRIME}/Fw/Python/docs/gendoc.bash"
     cd "${FPRIME}/Gds/docs"
     "${FPRIME}/Gds/docs/gendoc.bash"
 ) || exit 1
 
+# Fix for github pages
+github_page_adjustment "${DOXY_OUTPUT}/html"
+github_page_adjustment "${PY_OUTPUT}/fprime/html"
+github_page_adjustment "${PY_OUTPUT}/fprime-gds/html"

docs/doxygen/gh_pages.py

@@ -0,0 +1,72 @@
+""" Purge HTML directory of files with _ prefix
+
+Script to purge a directory of files starting with _ as this does not host well with GH pages and
+thus these files need to be rewritten. This file is designed for speed in these replacements.
+"""
+from __future__ import print_function, division # To work with any system python
+
+import os
+import re
+import shutil
+import sys
+
+from multiprocessing import Pool
+
+def converter(input_data):
+    """ Conversion function for thread pooling """
+    convert, changes  = input_data
+    def corrector(line):
+        """ Corrects all instances needing changing """
+        for fix, needle in changes:
+            line = line.replace(needle, fix)
+        return line
+    with open(convert, "r") as file_handle:
+        try:
+            read_lines = file_handle.readlines()
+        except UnicodeDecodeError:
+            if convert.endswith(".html") or convert.endswith(".js"):
+                raise
+            return
+    corrected = [corrector(line) for line in read_lines]
+    with open(convert, "w") as file_handle:
+        file_handle.write("".join(corrected))
+
+def absolution(all_files, changes):
+    """ Corrects all input files with the list of changed pairs """
+    total_len = len(all_files)
+    # Start a thread pool
+    data = zip(all_files, [changes] * total_len) 
+    pool = Pool()
+    for i, _ in enumerate(pool.imap_unordered(converter, list(data)), 1):
+        print("[INFO] Updating {}/{}                ".format(i, total_len), end="\r")
+        sys.stdout.flush()
+    print()
+
+
+def main():
+    """ Main program entry """
+    if len(sys.argv) < 2 or not os.path.isdir(sys.argv[1]):
+        print("[ERROR] Must supply a directory to correct", file=sys.stderr)
+        sys.exit(1)
+    directory = sys.argv[1]
+    all_files = os.listdir(directory)
+    # Detect broken files
+    defunct_files = [potential for potential in all_files if potential[0] == "_"]
+    corrected = [old[1:] for old in defunct_files]
+    # Move broken files
+    for mover, movie in zip(defunct_files, corrected):
+        shutil.move(os.path.join(directory, mover), os.path.join(directory, movie))
+    # Rewrite updates for above files
+    corrections_in = ['"{}'.format(correction) for correction in defunct_files] + \
+                     ['/{}'.format(correction) for correction in defunct_files]
+    corrections_out = ['"{}'.format(correction) for correction in corrected] + \
+                      ['/{}'.format(correction) for correction in corrected]
+    full_files = [os.path.join(path, fle) for path, _, files in os.walk(directory) for fle in files]
+    mappings = list(zip(corrections_out, corrections_in))
+    #for inin, outout in mappings:
+    #    print("{} -> {}".format(inin, outout))
+    absolution(full_files, mappings)
+
+
+if __name__ == "__main__":
+    main()