Add automatic manpage generation (#234)

- add manpage generation step to our custom Jekyll module that produces
manpage-compatible markdown files
- add manpage generation script to our GitHub action workflow that
converts the markdown files to real manpages, and stores the generated
manpages in the final site
- add back the dynamic version info usage in the produced site (and
therefore, in the produced manpages as well)

Author: HofiOne

.github/workflows/jekyll-gh-pages.yml

@@ -68,6 +68,10 @@ jobs:
           ./_tools/pack debug
           ls -AlR assets/js
 
+          # Update the site.config.version info in the _config.yml based on the current syslog-ng version
+          #
+          ./_tools/update_config_version
+
       - name: Build with Jekyll
         # Outputs to the './_site' directory by default
         run: |
@@ -83,20 +87,22 @@ jobs:
           JEKYLL_ENV: production
 
       - name: Run custom after build steps
-        # Seems jekyll build finds references from the minimal-mistakes bundle to the assets/js folder scripts
-        # even though most of them are packed in assets/js/main.min.js
-        # Re-packing only the required ones and overwrite the assets/js with the minimal result needed
-        # For more see the _tools/pack script
-        # TODO: Eliminate this later (probably building purely from local source the minimal-mistakes will solve this)
-        # TODO: Double-check if it's still needed
         run: |
+          # Seems jekyll build finds references from the minimal-mistakes bundle to the assets/js folder scripts
+          # even though most of them are packed in assets/js/main.min.js
+          # Re-packing only the required ones and overwrite the assets/js with the minimal result needed
+          # For more see the _tools/pack script
+          # TODO: Eliminate this later (probably building purely from local source the minimal-mistakes will solve this)
+          # TODO: Double-check if it's still needed
+          #
           echo "Updating main.min.js and re-pack js scripts"
           pwd
-
           ./_tools/pack debug
           ls -AlR _site/assets/js
 
           # Generate and add Doxygen source documentation
+          #
+          echo "Generating Doxygen source documentation"
           sudo apt-get install doxygen graphviz -y
 
           OSE_SRC_REPO=https://github.com/syslog-ng/syslog-ng.git
@@ -110,6 +116,13 @@ jobs:
           popd
           rm -Rf ./_work
 
+          # Generate final manpages from the pre-generated special markdown files
+          #
+          echo "Generating manpages"
+          INPUT_DIRECTORY=./_data/manpages
+          OUTPUT_DIRECTORY=./_site/manpages
+          ./_tools/mangen "${INPUT_DIRECTORY}" "${OUTPUT_DIRECTORY}"
+
       - name: Upload artifact
         # Automatically uploads an artifact from the './_site' directory by default
         uses: actions/upload-pages-artifact@v3

.gitignore

@@ -32,3 +32,4 @@ _data/links
 # _js/main.min.js
 assets/js
 _tools/package.json
+_data/manpages

Gemfile

@@ -96,6 +96,9 @@ gem "ostruct"
 # self made plugins dependencies
 gem "nokogiri"
 
+# https://github.com/sunaku/md2man?tab=readme-ov-file
+gem "md2man"
+
 # Gems loaded irrespective of site configuration.
 # If you have any other plugins, put them here!
 # Cf. https://jekyllrb.com/docs/plugins/installation/
@@ -111,6 +114,5 @@ group :jekyll_plugins do
     # https://github.com/HofiOne/minimal-mistakes
     #
     gem "jekyll-include-cache"
-	gem "jekyll-github-metadata"
     # gem "github-pages"
 end

_config.yml

@@ -76,6 +76,7 @@ exclude:
   #- assets/js
   - _js/
   - _tools/
+  - _data/manpages
   - Gemfile
   - Gemfile.lock
   - node_modules
@@ -257,5 +258,7 @@ footer:
 product:
   name: 'syslog-ng Open Source Edition'
   short_name: 'syslog-ng OSE'
+  # NOTE: This is updated automatically in the GitHub workflow (jekyll-gh-pages.yml) from the current syslog-ng version
+  version: '4.9.0'
 
 repository: syslog-ng/syslog-ng.github.io

_data/link_aliases.yml

@@ -3,6 +3,9 @@
 # Any not enclosed strings will be matched exactly like written.
 #
 
+adm-guide:
+  aliases: [ "/[Aa]dministration [Gg]uide/", "/[Aa]dministrator [Gg]uide/",  "/syslog-ng OSE [Aa]dministration [Gg]uide",  "/syslog-ng OSE [Aa]dministrator [Gg]uide"  ]
+
 adm-temp-macro-ose:
   aliases: [ "/[Mm]acros/" ]
 

_data/navigation.yml

@@ -1068,29 +1068,29 @@ admin-guide-nav:
     url: /admin-guide/190_The_syslog-ng_manual_pages/README
     subnav:
     - title: "The dqtool tool manual page"
-      url: /admin-guide/190_The_syslog-ng_manual_pages/000_dqtool
+      url: /admin-guide/190_The_syslog-ng_manual_pages/001_dqtool_manual
     - title: "The loggen manual page"
-      url: /admin-guide/190_The_syslog-ng_manual_pages/001_loggen
+      url: /admin-guide/190_The_syslog-ng_manual_pages/002_loggen_manual
     - title: "The pdbtool manual page"
-      url: /admin-guide/190_The_syslog-ng_manual_pages/002_pdbtool
+      url: /admin-guide/190_The_syslog-ng_manual_pages/003_pdbtool_manual
     - title: "The secure-logging manual page"
-      url: /admin-guide/190_The_syslog-ng_manual_pages/003_secure_logging_manual
+      url: /admin-guide/190_The_syslog-ng_manual_pages/004_secure_logging_manual
     - title: "The slogencrypt manual page"
-      url: /admin-guide/190_The_syslog-ng_manual_pages/004_slogencrypt_manual
+      url: /admin-guide/190_The_syslog-ng_manual_pages/005_slogencrypt_manual
     - title: "The slogkey manual page"
-      url: /admin-guide/190_The_syslog-ng_manual_pages/005_slogkey_manual
+      url: /admin-guide/190_The_syslog-ng_manual_pages/006_slogkey_manual
     - title: "The slogverify manual page"
-      url: /admin-guide/190_The_syslog-ng_manual_pages/006_slogverify_manual
+      url: /admin-guide/190_The_syslog-ng_manual_pages/007_slogverify_manual
     - title: "The {{ site.product.short_name }} control tool manual page"
-      url: /admin-guide/190_The_syslog-ng_manual_pages/007_syslog-ng_control_tool
+      url: /admin-guide/190_The_syslog-ng_manual_pages/008_syslog-ng_control_tool_manual
     - title: "The syslog-debun manual page"
-      url: /admin-guide/190_The_syslog-ng_manual_pages/008_syslog_debun_tool
+      url: /admin-guide/190_The_syslog-ng_manual_pages/009_syslog_debun_tool_manual
     - title: "The {{ site.product.short_name }} manual page"
-      url: /admin-guide/190_The_syslog-ng_manual_pages/009_syslog-ng_manual
+      url: /admin-guide/190_The_syslog-ng_manual_pages/010_syslog-ng_manual
     - title: "The syslog-ng.conf manual page"
-      url: /admin-guide/190_The_syslog-ng_manual_pages/010_syslog-ng_conf
+      url: /admin-guide/190_The_syslog-ng_manual_pages/011_syslog-ng_conf
     - title: "The persist-tool manual page"
-      url: /admin-guide/190_The_syslog-ng_manual_pages/011_persist-tool
+      url: /admin-guide/190_The_syslog-ng_manual_pages/012_persist-tool_manual
   - title: "About us"
     url: /admin-guide/200_About/README
     subnav:

_includes/doc/admin-guide/manpages-footnote.md

@@ -1,11 +1,19 @@
-## See also
+**NOTE:**
+If you experience any problems or need help with {{ site.product.short_name }}, see the *{{ site.product.short_name }} Administration Guide*[1], or visit the *{{ site.product.short_name }} mailing list*[2].
+For news and notifications about {{ site.product.short_name }}, visit the *{{ site.product.short_name }} blogs*[3].
+{: .notice--info}
 
-The syslog-ng.conf manual page  
-The {{ site.product.short_name }} manual page
+## AUTHOR
+This manual page was generated from the *{{ site.product.short_name }} Administration Guide*[1], which was written by several contributors to whom we'd like to extend our sincere thanks.
 
->**NOTE:**
->If you experience any problems or need help with {{ site.product.short_name }}, visit
->the {{ site.product.short_name }} mailing list.
->  
->For news and notifications about {{ site.product.short_name }}, visit the syslog-ng blogs.
-{: .notice--info}
+## COPYRIGHT
+## NOTES
+
+[1] `{{ site.product.short_name }} Administration Guide`
+    https://syslog-ng.github.io/admin-guide/README
+
+[2] `{{ site.product.short_name }} mailing list`
+    https://lists.balabit.hu/mailman/listinfo/syslog-ng
+
+[3] `{{ site.product.short_name }} blogs`
+    https://syslog-ng.com/blog/

_plugins/generate_tooltips.rb

@@ -2,6 +2,19 @@
 require 'liquid'
 
 module Jekyll
+  class SyslogNGTools
+    class << self
+      public
+
+        def write_to_file(file_path, content)
+          File.open(file_path, "w") do |file|
+            file.write(content)
+          end
+        end
+
+    end # self
+  end # SyslogNGTools
+
   class TooltipGen
     class << self
 
@@ -250,12 +263,6 @@ def process_page(page)
         page.content = parts.join
       end
 
-      def write_to_file(file_path, content)
-        File.open(file_path, "w") do |file|
-          file.write(content)
-        end
-      end
-
       def process_nav_link_items(items, ndx, nav_links_dictionary)
         items.each do |item|
           item['nav_ndx'] = ndx
@@ -397,14 +404,66 @@ def generate_tooltips(page, write_back)
         #puts "\n\n\n" + page.content
 
         if write_back
-          write_to_file(page.path, page.content)
+          SyslogNGTools.write_to_file(page.path, page.content)
         end
 
         puts "-----------"
       end # def generate_tooltips
 
     end # class << self
   end # class TooltipGen
+
+  class ManpageGen
+    class << self
+
+    private
+
+    public
+      def generate_manpage(page, manpages_dir)
+        puts "collection: " + (page.respond_to?(:collection) ? page.collection.label : "") + ", ndx: #{page.data["nav_ndx"]}, relative_path: #{page.relative_path}"
+
+        FileUtils.mkdir_p(manpages_dir)
+
+        page_id = page.data['id']
+        page_links = page.data["page_links"]
+        version_string = page.site.config['product']['version']
+        link_data = page_links[page_id]
+        title = link_data["title"][0]   # link_data["title"] is an array of titles that all must be represented by ID already in the filtered_page_ids_sorted_by_title_len array
+
+        # Remove the description rendering helper part from the manpage content
+        page_content = page.content
+        pattern = /^.*#{Regexp.escape(JekyllTooltipGen_description_start_tag)}.*$\n?/
+        page_content = page_content.gsub(pattern, '')
+        page_content = page_content.gsub(JekyllTooltipGen_description_end_tag, '')
+
+        # Compose a manpage header for md2man like
+        #
+        # manname manid "11 MARCH 1969" 4.9.0 "title"
+        # "======================================="
+        #
+        # "## NAME"
+        # description
+        #
+        date_str = Time.now.strftime("%d %B %Y")
+        manpage_header = "#{page.data["manname"]} #{page.data["manid"]} \"#{date_str}\" #{version_string} \"#{title}\"\n=======================================\n\n"
+        if page.data["description"] && false == page.data["description"].empty?
+          manpage_header = manpage_header + "## NAME\n" + page.data["description"] + "\n\n"
+        end
+
+        page_content = manpage_header + page_content
+        # TODO: Check why these are not rendered yet
+        # Remove some special, not yet rendered liquid, markdown notations, we do not want in the manpage output either
+        page_content = page_content.gsub(/\{\:[^}]*\}/, "")
+
+        outfile = File.join(manpages_dir, File.basename(page.path))
+        SyslogNGTools.write_to_file(outfile, page_content)
+
+        puts "-----------"
+      end # def generate_tooltips
+
+  end # class << self
+  end # ManpageGen
+
 end # module jekyll
 
 def JekyllTooltipGen_debug_page_info(page, details = true)
@@ -485,7 +544,7 @@ def JekyllTooltipGen_hack_description_in(page_has_subtitle, page_has_description
 # This is used now to
 #       - set the page nav_ndx correctly to support our custom bottom collection elements navigator
 #       - set additional page data elements that will be used during all the passes
-#       - add the description to the beginning of the page.content to get it rendred correclty the same way, together with the page content
+#       - add the description to the beginning of the page.content to get it rendered correctly the same way, together with the page content
 #
 # NOTE: Do not use this site based enumeration directly for the page content manipulation as well
 #       as that needs proper per-page payload data (or TODO: figure out how to get it in that case properly)
@@ -532,10 +591,13 @@ def JekyllTooltipGen_hack_description_in(page_has_subtitle, page_has_description
   end
 end
 
+JekyllManpageGen_manpages_folder = '_data/manpages'
+
 # 3rd pass
 #
 # This is used now to
 #     - render the page content manually and create the autolinks and tooltips
+#     - create manpage input markdown files for the manual pages
 #
 Jekyll::Hooks.register [:pages, :documents], :pre_render do |page, payload|
   next if false == $JekyllTooltipGen_should_build_tooltips
@@ -555,8 +617,16 @@ def JekyllTooltipGen_hack_description_in(page_has_subtitle, page_has_description
   }
   page.content = template.render!(payload, info)
 
-  Jekyll::TooltipGen.generate_tooltips(page, $JekyllTooltipGen_should_build_persistent_tooltips)
+  # Generate a manpage input markdown file if manid is defined in the page front-matter
+  if page.data['manid']
+    if page.data["manname"] == nil
+      puts "Error: manid found without manname in page: #{page.relative_path}"
+      exit 5
+    end
+    Jekyll::ManpageGen.generate_manpage(page, JekyllManpageGen_manpages_folder)
+  end
 
+  Jekyll::TooltipGen.generate_tooltips(page, $JekyllTooltipGen_should_build_persistent_tooltips)
   page.content = page.content.gsub(JekyllTooltipGen_description_start_tag, '<p id="page-description">')
   page.content = page.content.gsub(JekyllTooltipGen_description_end_tag, '</p>')
 end

_tools/mangen

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+process_params()
+{
+    mkdir -p ${OUTPUT_DIRECTORY} || true
+
+    for FILE in $(/bin/ls ${INPUT_DIRECTORY}/*.md); do
+        echo "Processing ${FILE} file..."
+        BASENAME=$(basename "${FILE}")
+        OUTPUT_FILE_TYPE=$([[ $BASENAME == *_config ]] && echo 5 || echo 1)
+        MANNAME=$(head -n 1 "${FILE}" | awk '{print $1}')
+
+        bundle exec md2man-roff ${FILE} > ${OUTPUT_DIRECTORY}/${MANNAME}.${OUTPUT_FILE_TYPE}
+    done
+
+}
+
+
+# Check for the correct number of command-line parameters
+if [ "$#" -lt 2 ]; then
+    echo "Usage: $0 <input_folder> <output_folder>"
+    exit 1
+fi
+
+INPUT_DIRECTORY="${1}"
+OUTPUT_DIRECTORY="${2}"
+
+process_params "$@"
+
+exit 0

_tools/update_config_version

@@ -0,0 +1,9 @@
+echo "Updating version info"
+
+VERSION=$(curl -s https://raw.githubusercontent.com/syslog-ng/syslog-ng/develop/VERSION.txt)
+sed -E "s/(version: ')[0-9]+\.[0-9]+\.[0-9]+(')/\1$VERSION\2/" _config.yml > _config.yml.new
+
+mv -f _config.yml.new _config.yml
+
+echo "Version set to $VERSION"
+grep 'version:' _config.yml