From 013e4009ccec821b1ab6631811cf3a70b5a95d91 Mon Sep 17 00:00:00 2001 From: sdarwin Date: Tue, 10 Mar 2026 18:02:23 +0100 Subject: [PATCH 1/2] build_docs: macosdocs.sh follows linuxdocs.sh --- .github/workflows/build_docs.yml | 29 +++++++++++++++----- build_docs/linuxdocs.sh | 2 +- build_docs/macosdocs.sh | 45 +++++++++++++++++++++++--------- build_docs/windowsdocs.ps1 | 1 + 4 files changed, 57 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml index cf667b3..4c778b5 100644 --- a/.github/workflows/build_docs.yml +++ b/.github/workflows/build_docs.yml @@ -20,6 +20,9 @@ on: - develop - feature/** +env: + NODEJS_LIBRARIES: "decimal redis unordered url" + jobs: linux: defaults: @@ -113,10 +116,10 @@ jobs: linuxdocs.sh ${{ matrix.flags }} cd ../.. - # Run at least one antora build that installs everything - cd libs/url - linuxdocs.sh ${{ matrix.flags }} - cd ../.. + # # Run at least one antora build that installs everything + # cd libs/url + # linuxdocs.sh ${{ matrix.flags }} + # cd ../.. textpart1='#!/bin/bash reponame=$1 @@ -133,6 +136,11 @@ jobs: echo "skipping ahead X letters" elif [[ "$skiplist" =~ $reponame ]]; then echo "repo in skiplist" + elif [[ "$NODEJS_LIBRARIES" =~ $reponame ]]; then + # The antora submodule problem. We would need to + # run git init. Skipping for now. + true + # linuxdocs.sh --skip-boost else linuxdocs.sh --quick ' @@ -243,6 +251,11 @@ jobs: echo "skipping ahead X letters" elif [[ "$skiplist" =~ $reponame ]]; then echo "repo in skiplist" + elif [[ "$NODEJS_LIBRARIES" =~ $reponame ]]; then + # The antora submodule problem. We would need to + # run git init. Skipping for now. + true + # macosdocs.sh --skip-boost else macosdocs.sh --quick ' @@ -358,6 +371,12 @@ jobs: elseif ($skiplist -like "*$reponame*") { echo "repo in skiplist" } + elseif ($NODEJS_LIBRARIES -like "*$reponame*") { + # The antora submodule problem. We would need to + # run git init. Skipping for now. + $true + # windowsdocs.ps1 -skip-boost + } else { windowsdocs.ps1 -quick '@ @@ -415,5 +434,3 @@ jobs: if ( $failed -ne '0' ) { exit 1 } - - diff --git a/build_docs/linuxdocs.sh b/build_docs/linuxdocs.sh index 4971fed..1f291ac 100755 --- a/build_docs/linuxdocs.sh +++ b/build_docs/linuxdocs.sh @@ -255,7 +255,7 @@ if [ "$skippackagesoption" != "yes" ]; then sudo gem install asciidoctor-pdf --version 2.3.4 sudo gem install asciidoctor-diagram --version 2.2.14 sudo gem install asciidoctor-multipage --version 0.0.18 - pip3 install setuptools + pip3 install setuptools==75.6.0 pip3 install docutils # which library is using rapidxml # wget -O rapidxml.zip http://sourceforge.net/projects/rapidxml/files/latest/download diff --git a/build_docs/macosdocs.sh b/build_docs/macosdocs.sh index d665058..6ec5f37 100755 --- a/build_docs/macosdocs.sh +++ b/build_docs/macosdocs.sh @@ -249,12 +249,21 @@ echo '==================================> INSTALL' if [ "$skippackagesoption" != "yes" ]; then if [ "$typeoption" = "antora" ] || [ "$install_antora_deps" = "yes" ]; then - mkdir -p ~/".nvm_${REPONAME}_antora" + # A shared nvm installation potentially exists in /opt/nvm + # and it saves disk space and downloads. + + if [ -d "/opt/nvm" ]; then + export NVM_DIR=/opt/nvm + echo "Use shared nvm installation" + else + export NVM_DIR=$HOME/.nvm_${REPONAME}_antora + echo "Use home dir nvm installation" + fi + mkdir -p "$NVM_DIR" export NODE_VERSION=18.18.1 # The container has a pre-installed nodejs. Overwrite those again. - export NVM_BIN="$HOME/.nvm_${REPONAME}_antora/versions/node/v${NODE_VERSION}/bin" - export NVM_DIR=$HOME/.nvm_${REPONAME}_antora - export NVM_INC=$HOME/.nvm_${REPONAME}_antora/versions/node/v${NODE_VERSION}/include/node + export NVM_BIN="$NVM_DIR/versions/node/v${NODE_VERSION}/bin" + export NVM_INC=$NVM_DIR/versions/node/v${NODE_VERSION}/include/node curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash export NVM_DIR=$HOME/.nvm_${REPONAME}_antora # shellcheck source=/dev/null @@ -266,9 +275,13 @@ if [ "$skippackagesoption" != "yes" ]; then export PATH="$HOME/.nvm_${REPONAME}_antora/versions/node/v${NODE_VERSION}/bin/:${PATH}" node --version npm --version - npm install gulp-cli@2.3.0 - if grep -r mermaid "$BOOST_SRC_FOLDER/doc/"; then - npm install @mermaid-js/mermaid-cli@10.5.1 + if ! which gulp; then + npm install -g gulp-cli@2.3.0 + fi + if ! which mmdc; then + if grep -r mermaid "$BOOST_SRC_FOLDER/doc/"; then + npm install -g @mermaid-js/mermaid-cli@10.5.1 + fi fi fi @@ -321,7 +334,7 @@ if [ "$skippackagesoption" != "yes" ]; then sudo gem install asciidoctor-pdf --version 2.3.4 sudo gem install asciidoctor-diagram --version 2.2.14 sudo gem install asciidoctor-multipage --version 0.0.18 - pip3 install setuptools + pip3 install setuptools==75.6.0 pip3 install docutils # which library is using rapidxml # wget -O rapidxml.zip http://sourceforge.net/projects/rapidxml/files/latest/download @@ -369,15 +382,21 @@ if [ -f "${pythonvirtenvpath}/bin/activate" ]; then fi # In the above 'packages' section npm was installed. Activate it, if that has not been done already. -if [ -d "$HOME/.nvm_${REPONAME}_antora" ]; then +if [ -d "/opt/nvm" ]; then + export NVM_DIR=/opt/nvm + echo "Use shared nvm installation" +else + export NVM_DIR=$HOME/.nvm_${REPONAME}_antora + echo "Use home dir nvm installation" +fi +if [ -d "$NVM_DIR" ]; then export NODE_VERSION=18.18.1 # The container has a pre-installed nodejs. Overwrite those again. - export NVM_BIN="$HOME/.nvm_${REPONAME}_antora/versions/node/v${NODE_VERSION}/bin" - export NVM_DIR=$HOME/.nvm_${REPONAME}_antora - export NVM_INC=$HOME/.nvm_${REPONAME}_antora/versions/node/v${NODE_VERSION}/include/node + export NVM_BIN="$NVM_DIR/versions/node/v${NODE_VERSION}/bin" + export NVM_INC=$NVM_DIR/versions/node/v${NODE_VERSION}/include/node # shellcheck source=/dev/null . "$NVM_DIR/nvm.sh" && nvm use "v${NODE_VERSION}" - export PATH="$HOME/.nvm_${REPONAME}_antora/versions/node/v${NODE_VERSION}/bin/:${PATH}" + export PATH="$NVM_DIR/versions/node/v${NODE_VERSION}/bin/:${PATH}" node --version npm --version fi diff --git a/build_docs/windowsdocs.ps1 b/build_docs/windowsdocs.ps1 index bee11bd..6fd1eca 100755 --- a/build_docs/windowsdocs.ps1 +++ b/build_docs/windowsdocs.ps1 @@ -455,6 +455,7 @@ if ( -Not ${skip-packages} ) { gem install pygments.rb --version 2.3.0 pip3 install Pygments==2.13.0 gem install rouge --version 4.0.0 + pip3 install setuptools==75.6.0 pip3 install Sphinx==5.2.1 pip3 install --user git+https://github.com/pfultz2/sphinx-boost@8ad7d424c6b613864976546d801439c34a27e3f6 # from dockerfile: From 0d5b6a04f30818cc80317dee2fb6b685a2ab20f5 Mon Sep 17 00:00:00 2001 From: sdarwin Date: Wed, 11 Mar 2026 19:27:45 +0100 Subject: [PATCH 2/2] build_docs: Install gems from Gemfile, pip packages from requirements.txt --- .github/workflows/build_docs.yml | 35 +++++---------- build_docs/linuxdocs.sh | 75 +++++++++++++++++++++----------- build_docs/macosdocs.sh | 75 ++++++++++++++++++++++---------- build_docs/windowsdocs.ps1 | 73 ++++++++++++++++++++----------- 4 files changed, 158 insertions(+), 100 deletions(-) diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml index 4c778b5..c7f91c8 100644 --- a/.github/workflows/build_docs.yml +++ b/.github/workflows/build_docs.yml @@ -44,11 +44,6 @@ jobs: packages: python2 skiplist: auto_index geometry url flags: --debug - - os: ubuntu-latest - container: ubuntu:20.04 - packages: python - skiplist: auto_index geometry url - flags: --debug - os: ubuntu-latest container: ubuntu:24.04 packages: python-is-python3 @@ -59,12 +54,6 @@ jobs: packages: python2 skiplist: geometry url flags: --debug --boostrelease - - os: ubuntu-latest - container: ubuntu:20.04 - packages: python - skiplist: geometry url - flags: --debug --boostrelease - # Jinja 3.1.0 and later have dropped support for Python 3.6 and therefore won't run on Ubuntu 18.04. Removing tests for Ubuntu 18. timeout-minutes: 720 runs-on: ${{matrix.os}} @@ -182,21 +171,16 @@ jobs: fail-fast: false matrix: include: - # - os: macos-11 - # skiplist: auto_index contract - # - os: macos-11 - # skiplist: contract - # flags: --debug --boostrelease - - os: macos-12 + - os: macos-14 skiplist: auto_index contract geometry url flags: --debug - - os: macos-12 + - os: macos-14 skiplist: contract geometry url flags: --debug --boostrelease - - os: macos-14 + - os: macos-15 skiplist: auto_index contract geometry url flags: --debug - - os: macos-14 + - os: macos-15 skiplist: contract geometry url flags: --debug --boostrelease @@ -295,14 +279,15 @@ jobs: fail-fast: false matrix: include: - - os: windows-2019 - skiplist: auto_index contract hana leaf log parameter parameter_python python qvm url - os: windows-2022 skiplist: auto_index contract hana log parameter parameter_python python url - - os: windows-2019 + - os: windows-2025 + skiplist: auto_index contract hana log parameter parameter_python python url + - os: windows-2022 + lint: yes flags: "-boostrelease" skiplist: contract hana log parameter parameter_python python url - - os: windows-2022 + - os: windows-2025 lint: yes flags: "-boostrelease" skiplist: contract hana log parameter parameter_python python url @@ -371,7 +356,7 @@ jobs: elseif ($skiplist -like "*$reponame*") { echo "repo in skiplist" } - elseif ($NODEJS_LIBRARIES -like "*$reponame*") { + elseif ($env:NODEJS_LIBRARIES -like "*$reponame*") { # The antora submodule problem. We would need to # run git init. Skipping for now. $true diff --git a/build_docs/linuxdocs.sh b/build_docs/linuxdocs.sh index 1f291ac..645aa62 100755 --- a/build_docs/linuxdocs.sh +++ b/build_docs/linuxdocs.sh @@ -16,6 +16,43 @@ boostrelease="" BOOSTROOTRELPATH=".." pythonvirtenvpath="${HOME}/venvboostdocs" +requirements_txt=" +Jinja2==3.1.4 +MarkupSafe==3.0.2 +Pygments==2.18.0 +Sphinx==5.2.1 +docutils==0.19 +future==1.0.0 +git+https://github.com/pfultz2/sphinx-boost@8ad7d424c6b613864976546d801439c34a27e3f6 +https://github.com/bfgroup/jam_pygments/archive/master.zip +myst-parser==0.18.1 +setuptools==75.6.0 +six==1.17.0 +" + +gemfile=" +source 'http://rubygems.org' +gem 'asciidoctor',' 2.0.23' +gem 'asciidoctor-diagram', '2.3.1' +gem 'asciidoctor-multipage', '0.0.19' +gem 'rouge', '4.5.1' +gem 'pygments.rb', '3.0.0' +" + +# Are there some gems we are installing in build_docs which aren't installed in release-tools? +# Keep them in gemfile_local +# +# Earlier notes about public_suffix and css_parser: +# # 1.12.0 from 2022 still supports ruby 2.5. Continue to use until ~2024. +# So maybe this can be removed soon. +gemfile_local=" +source 'http://rubygems.org' +gem 'public_suffix', '4.0.7' +gem 'css_parser', '1.12.0' +gem 'afm', '0.2.2' +gem 'asciidoctor-pdf', '2.3.4' +" + # READ IN COMMAND-LINE OPTIONS TEMP=$(getopt -o t:,h::,q:: --long type:,help::,skip-boost::,skip-packages::,quick::,boostrelease::,boostrootsubdir::,debug:: -- "$@") @@ -247,31 +284,19 @@ if [ "$skippackagesoption" != "yes" ]; then # shellcheck source=/dev/null source "${pythonvirtenvpath}/bin/activate" - # the next two gems are for asciidoctor-pdf - sudo gem install public_suffix --version 4.0.7 # 4.0.7 from 2022 still supports ruby 2.5. Continue to use until ~2024. - sudo gem install css_parser --version 1.12.0 # 1.12.0 from 2022 still supports ruby 2.5. Continue to use until ~2024. - sudo gem install afm --version 0.2.2 - sudo gem install asciidoctor --version 2.0.17 - sudo gem install asciidoctor-pdf --version 2.3.4 - sudo gem install asciidoctor-diagram --version 2.2.14 - sudo gem install asciidoctor-multipage --version 0.0.18 - pip3 install setuptools==75.6.0 - pip3 install docutils - # which library is using rapidxml - # wget -O rapidxml.zip http://sourceforge.net/projects/rapidxml/files/latest/download - # unzip -n -d rapidxml rapidxml.zip - pip3 install https://github.com/bfgroup/jam_pygments/archive/master.zip - pip3 install Jinja2==3.1.2 - pip3 install MarkupSafe==2.1.1 - sudo gem install pygments.rb --version 2.3.0 - pip3 install Pygments==2.13.0 - sudo gem install rouge --version 4.0.0 - pip3 install Sphinx==5.2.1 - pip3 install git+https://github.com/pfultz2/sphinx-boost@8ad7d424c6b613864976546d801439c34a27e3f6 - # from dockerfile: - pip3 install myst-parser==0.18.1 - pip3 install future==0.18.2 - pip3 install six==1.14.0 + # Install pip packages + mkdir -p /tmp/build_docs + echo "$requirements_txt" > /tmp/build_docs/requirements.txt + pip3 install -r /tmp/build_docs/requirements.txt + rm /tmp/build_docs/requirements.txt + + # Install ruby gems + gem install bundler + echo "$gemfile" > /tmp/build_docs/Gemfile + BUNDLE_GEMFILE=/tmp/build_docs/Gemfile bundle install + echo "$gemfile_local" > /tmp/build_docs/Gemfile + BUNDLE_GEMFILE=/tmp/build_docs/Gemfile bundle install + rm /tmp/build_docs/Gemfile # Locking the version numbers in place offers a better guarantee of a known, good build. # At the same time, it creates a perpetual outstanding task, to upgrade the gem and pip versions diff --git a/build_docs/macosdocs.sh b/build_docs/macosdocs.sh index 6ec5f37..ec040ae 100755 --- a/build_docs/macosdocs.sh +++ b/build_docs/macosdocs.sh @@ -24,6 +24,43 @@ boostrelease="" BOOSTROOTRELPATH=".." pythonvirtenvpath="${HOME}/venvboostdocs" +requirements_txt=" +Jinja2==3.1.4 +MarkupSafe==3.0.2 +Pygments==2.18.0 +Sphinx==5.2.1 +docutils==0.19 +future==1.0.0 +git+https://github.com/pfultz2/sphinx-boost@8ad7d424c6b613864976546d801439c34a27e3f6 +https://github.com/bfgroup/jam_pygments/archive/master.zip +myst-parser==0.18.1 +setuptools==75.6.0 +six==1.17.0 +" + +gemfile=" +source 'http://rubygems.org' +gem 'asciidoctor',' 2.0.23' +gem 'asciidoctor-diagram', '2.3.1' +gem 'asciidoctor-multipage', '0.0.19' +gem 'rouge', '4.5.1' +gem 'pygments.rb', '3.0.0' +" + +# Are there some gems we are installing in build_docs which aren't installed in release-tools? +# Keep them in gemfile_local +# +# Earlier notes about public_suffix and css_parser: +# # 1.12.0 from 2022 still supports ruby 2.5. Continue to use until ~2024. +# So maybe this can be removed soon. +gemfile_local=" +source 'http://rubygems.org' +gem 'public_suffix', '4.0.7' +gem 'css_parser', '1.12.0' +gem 'afm', '0.2.2' +gem 'asciidoctor-pdf', '2.3.4' +" + # git and getopt are required. If they are not installed, moving that part of the installation process # to an earlier part of the script: if ! command -v brew &> /dev/null @@ -327,30 +364,20 @@ if [ "$skippackagesoption" != "yes" ]; then brew install ghostscript brew install texlive brew install graphviz - sudo gem install public_suffix --version 4.0.7 # 4.0.7 from 2022 still supports ruby 2.5. Continue to use until ~2024. - sudo gem install css_parser --version 1.12.0 # 1.12.0 from 2022 still supports ruby 2.5. Continue to use until ~2024. - sudo gem install asciidoctor --version 2.0.17 - sudo gem install afm --version 0.2.2 - sudo gem install asciidoctor-pdf --version 2.3.4 - sudo gem install asciidoctor-diagram --version 2.2.14 - sudo gem install asciidoctor-multipage --version 0.0.18 - pip3 install setuptools==75.6.0 - pip3 install docutils - # which library is using rapidxml - # wget -O rapidxml.zip http://sourceforge.net/projects/rapidxml/files/latest/download - # unzip -n -d rapidxml rapidxml.zip - pip3 install https://github.com/bfgroup/jam_pygments/archive/master.zip - pip3 install Jinja2==3.1.2 - pip3 install MarkupSafe==2.1.1 - sudo gem install pygments.rb --version 2.3.0 - pip3 install Pygments==2.13.0 - sudo gem install rouge --version 4.0.0 - pip3 install Sphinx==5.2.1 - pip3 install git+https://github.com/pfultz2/sphinx-boost@8ad7d424c6b613864976546d801439c34a27e3f6 - # from dockerfile: - pip3 install myst-parser==0.18.1 - pip3 install future==0.18.2 - pip3 install six==1.14.0 + + # Install pip packages + mkdir -p /tmp/build_docs + echo "$requirements_txt" > /tmp/build_docs/requirements.txt + pip3 install -r /tmp/build_docs/requirements.txt + rm /tmp/build_docs/requirements.txt + + # Install ruby gems + gem install bundler + echo "$gemfile" > /tmp/build_docs/Gemfile + BUNDLE_GEMFILE=/tmp/build_docs/Gemfile bundle install + echo "$gemfile_local" > /tmp/build_docs/Gemfile + BUNDLE_GEMFILE=/tmp/build_docs/Gemfile bundle install + rm /tmp/build_docs/Gemfile # Locking the version numbers in place offers a better guarantee of a known, good build. # At the same time, it creates a perpetual outstanding task, to upgrade the gem and pip versions diff --git a/build_docs/windowsdocs.ps1 b/build_docs/windowsdocs.ps1 index 6fd1eca..203c13e 100755 --- a/build_docs/windowsdocs.ps1 +++ b/build_docs/windowsdocs.ps1 @@ -22,6 +22,37 @@ $nvm_install_version="1.1.11" $node_version="20.17.0" $node_version_basic="20" +$requirements_txt = @" +Jinja2==3.1.4 +MarkupSafe==3.0.2 +Pygments==2.18.0 +Sphinx==5.2.1 +docutils==0.19 +future==1.0.0 +git+https://github.com/pfultz2/sphinx-boost@8ad7d424c6b613864976546d801439c34a27e3f6 +https://github.com/bfgroup/jam_pygments/archive/master.zip +myst-parser==0.18.1 +setuptools==75.6.0 +six==1.17.0 +"@ + +$gemfile = @" +source 'http://rubygems.org' +gem 'asciidoctor',' 2.0.23' +gem 'asciidoctor-diagram', '2.3.1' +gem 'asciidoctor-multipage', '0.0.19' +gem 'rouge', '4.5.1' +gem 'pygments.rb', '3.0.0' +"@ + +$gemfile_local = @" +source 'http://rubygems.org' +gem 'public_suffix', '4.0.7' +gem 'css_parser', '1.12.0' +gem 'afm', '0.2.2' +gem 'asciidoctor-pdf', '2.3.4' +"@ + # Set-PSDebug -Trace 1 if ($help) { @@ -436,32 +467,22 @@ if ( -Not ${skip-packages} ) { choco install -y --no-progress ghostscript --version $ghostversion choco install -y --no-progress texlive choco install -y --no-progress graphviz - gem install public_suffix --version 4.0.7 # 4.0.7 from 2022 still supports ruby 2.5. Continue to use until ~2024. - gem install css_parser --version 1.12.0 # 1.12.0 from 2022 still supports ruby 2.5. Continue to use until ~2024. - gem install afm --version 0.2.2 - gem install asciidoctor --version 2.0.17 - gem install asciidoctor-pdf --version 2.3.4 - gem install asciidoctor-diagram --version 2.2.14 - gem install asciidoctor-multipage --version 0.0.18 - pip3 install docutils - ## Invoke-WebRequest -O rapidxml.zip http://sourceforge.net/projects/rapidxml/files/latest/download - # Invoke-WebRequest -O rapidxml.zip https://downloads.sourceforge.net/project/rapidxml/rapidxml/rapidxml%201.13/rapidxml-1.13.zip - # unzip -n -d rapidxml rapidxml.zip - # - # pip3 had been using --user. what will happen without. - pip3 install https://github.com/bfgroup/jam_pygments/archive/master.zip - pip3 install Jinja2==3.1.2 - pip3 install MarkupSafe==2.1.1 - gem install pygments.rb --version 2.3.0 - pip3 install Pygments==2.13.0 - gem install rouge --version 4.0.0 - pip3 install setuptools==75.6.0 - pip3 install Sphinx==5.2.1 - pip3 install --user git+https://github.com/pfultz2/sphinx-boost@8ad7d424c6b613864976546d801439c34a27e3f6 - # from dockerfile: - pip3 install myst-parser==0.18.1 - pip3 install future==0.18.2 - pip3 install six==1.14.0 + + # Install pip packages + New-Item -ItemType Directory -Force -Path ${HOME}/tmp/build_docs | Out-Null + $requirements_txt | Set-Content ${HOME}/tmp/build_docs/requirements.txt + pip3 install -r ${HOME}/tmp/build_docs/requirements.txt + Remove-Item ${HOME}/tmp/build_docs/requirements.txt + + # Install ruby gems + gem install bundler + $gemfile | Set-Content ${HOME}/tmp/build_docs/Gemfile + $env:BUNDLE_GEMFILE="${HOME}/tmp/build_docs/Gemfile" + bundle install + $gemfile_local | Set-Content ${HOME}/tmp/build_docs/Gemfile + $env:BUNDLE_GEMFILE="${HOME}/tmp/build_docs/Gemfile" + bundle install + Remove-Item ${HOME}/tmp/build_docs/Gemfile refenv