Support Ubuntu 20.04 packages with OnDemand 2.0 (#2141)

* Support Ubuntu 20.04 packages
* Use 2.0 release package to bootstrap E2E tests
* Backport Ubuntu support into ood-portal-generator
* Backport htcacheclean restart removal
* Ensure /opt/ood/ondemand/enable is loaded by Ubuntu packaged OnDemand
* Also handle doc labels for release branches
* Bump ood_packaging dependency
* Run update_ood_portal during postinst to avoid changing config files during postinst that are also part of package
* Update beaker-docker dependency
* Do not install package or test gems
* Fix logic to avoid issues when re-running tests
* Support Ubuntu with nginx_stage
* rnode location edits account for relative paths (#1934)

Enable rnode location edits account for relative paths because they broke in apache 2.4. This has the explicit group for them now.

Co-authored-by: Jeff Ohrstrom <johrstrom@osc.edu>

Author: treydock

.github/workflows/document-merge.yml

@@ -6,6 +6,7 @@ on:
     branches:
       - master
       - 'feature/**'
+      - 'release_**'
     types: [closed]
 
 jobs:

.github/workflows/tests.yml

@@ -78,7 +78,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dist: ["el7", "el8"]
+        dist: ["el7", "el8", "ubuntu-20.04"]
         version: ["2.0"]
     runs-on: "ubuntu-latest"
     name: E2E test ${{ matrix.dist }} (ondemand-${{ matrix.version }})

.gitlab-ci.yml

@@ -20,7 +20,7 @@ build:
     - bundle exec rake package:build[$OOD_PACKAGING_DIST]
   parallel:
     matrix:
-      - OOD_PACKAGING_DIST: [el7, el8]
+      - OOD_PACKAGING_DIST: [el7, el8, ubuntu-20.04]
   artifacts:
     paths:
       - dist

Gemfile

@@ -6,9 +6,10 @@ git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
 
 gem "rake"
 gem "bcrypt"
+gem 'dotenv', '~> 2.1'
 
 group :package do
-  gem 'ood_packaging', '0.0.1.r2.0'
+  gem 'ood_packaging', '0.0.2.r2.0'
 end
 
 group :test do
@@ -18,8 +19,5 @@ group :test do
 
   gem "beaker"
   gem "beaker-rspec"
-  # Use fork until merged and released
-  # https://github.com/voxpupuli/beaker-docker/pull/53
-  # https://github.com/voxpupuli/beaker-docker/pull/54
-  gem "beaker-docker", git: 'https://github.com/treydock/beaker-docker.git', branch: 'osc'
+  gem "beaker-docker", '~> 1.1.1'
 end

Gemfile.lock

@@ -1,20 +1,12 @@
-GIT
-  remote: https://github.com/treydock/beaker-docker.git
-  revision: a4c8f255db29903ada6e780872fd873bbf93303c
-  branch: osc
-  specs:
-    beaker-docker (1.0.0)
-      docker-api (~> 2.1)
-      stringify-hash (~> 0.0.0)
-
 GEM
   remote: https://rubygems.org/
   specs:
     ansi (1.5.0)
     ast (2.4.2)
     bcrypt (3.1.16)
-    beaker (4.30.0)
+    beaker (4.37.0)
       beaker-hostgenerator
+      ed25519 (~> 1.0)
       hocon (~> 1.0)
       in-parallel (~> 0.1)
       inifile (~> 3.0)
@@ -36,9 +28,12 @@ GEM
       hocon (~> 1.0)
       require_all (>= 1.3.2, < 3.1.0)
       stringify-hash (~> 0.0.0)
-    beaker-hostgenerator (1.5.0)
-      deep_merge (~> 1.0)
+    beaker-docker (1.1.1)
+      beaker (>= 4.34)
+      docker-api (~> 2.1)
       stringify-hash (~> 0.0.0)
+    beaker-hostgenerator (1.16.0)
+      deep_merge (~> 1.0)
     beaker-pe (2.11.13)
       beaker (~> 4.0)
       beaker-abs
@@ -65,12 +60,14 @@ GEM
     colorize (0.8.1)
     commander (4.6.0)
       highline (~> 2.0.0)
-    deep_merge (1.2.1)
+    deep_merge (1.2.2)
     diff-lcs (1.4.4)
     docker-api (2.2.0)
       excon (>= 0.47.0)
       multi_json
-    excon (0.85.0)
+    dotenv (2.7.6)
+    ed25519 (1.3.0)
+    excon (0.92.3)
     faraday (1.7.0)
       faraday-em_http (~> 1.0)
       faraday-em_synchrony (~> 1.0)
@@ -96,17 +93,17 @@ GEM
     inifile (3.0.0)
     method_source (1.0.0)
     minitar (0.9)
-    minitest (5.14.4)
+    minitest (5.16.2)
     multi_json (1.15.0)
     multipart-post (2.1.1)
-    net-scp (3.0.0)
-      net-ssh (>= 2.6.5, < 7.0.0)
-    net-ssh (6.1.0)
+    net-scp (1.2.1)
+      net-ssh (>= 2.6.5)
+    net-ssh (7.0.1)
     net-telnet (0.1.1)
     oga (3.3)
       ast
       ruby-ll (~> 2.1)
-    ood_packaging (0.0.1.r2.0)
+    ood_packaging (0.0.2.r2.0)
       rake (~> 13.0.1)
     open_uri_redirections (0.2.1)
     parallel (1.20.1)
@@ -173,7 +170,7 @@ GEM
       net-telnet (= 0.1.1)
       sfl
     stringify-hash (0.0.2)
-    thor (1.1.0)
+    thor (1.2.1)
     unicode-display_width (2.0.0)
     vmfloaty (1.4.0)
       colorize (~> 0.8.1)
@@ -189,9 +186,10 @@ PLATFORMS
 DEPENDENCIES
   bcrypt
   beaker
-  beaker-docker!
+  beaker-docker (~> 1.1.1)
   beaker-rspec
-  ood_packaging (= 0.0.1.r2.0)
+  dotenv (~> 2.1)
+  ood_packaging (= 0.0.2.r2.0)
   rake
   rspec
   rubocop

Rakefile

@@ -5,18 +5,21 @@ PROJ_DIR          = Pathname.new(__dir__)
 TASK_DIR          = "#{PROJ_DIR}/lib/tasks"
 APPS_DIR          = PROJ_DIR.join('apps')
 GEMFILE           = PROJ_DIR.join('Gemfile')
-INSTALL_ROOT      = Pathname.new(ENV["PREFIX"] || "/opt/ood")
+DESTDIR           = Pathname.new(ENV['DESTDIR'].to_s)
+INSTALL_ROOT      = Pathname.new(ENV["PREFIX"] || "#{DESTDIR}/opt/ood")
 VENDOR_BUNDLE     = (ENV['VENDOR_BUNDLE'] == "yes" || ENV['VENDOR_BUNDLE'] == "true")
 PASSENGER_APP_ENV = ENV["PASSENGER_APP_ENV"] || "production"
 
 require "#{TASK_DIR}/rake_helper"
+require "#{TASK_DIR}/build_utils"
 require "#{TASK_DIR}/packaging"
 require "#{TASK_DIR}/test"
 require "#{TASK_DIR}/lint"
 require "#{TASK_DIR}/docker"
 require "#{TASK_DIR}/development"
 
 include RakeHelper
+include BuildUtils
 
 namespace :build do
   desc "Build gems"
@@ -77,8 +80,20 @@ namespace :install do
     sh "cp -r #{APPS_DIR} #{INSTALL_ROOT}/"
   end
 
+  namespace :infrastructure do
+    desc 'Install infrastructure files'
+    task :files do
+      infrastructure_files.each do |file|
+        src = render_package_file(file[:src])
+        FileUtils.mkdir_p(File.dirname(file[:dest]), verbose: true) unless Dir.exist?(File.dirname(file[:dest]))
+        FileUtils.cp(src, file[:dest], verbose: true)
+        FileUtils.chmod(file[:mode], file[:dest], verbose: true)
+      end
+    end
+  end
+
   desc "Install OnDemand infrastructure and apps"
-  task :all => [:infrastructure, :apps]
+  task :all => [:infrastructure, :apps, 'infrastructure:files']
 end
 
 desc "Install OnDemand"

lib/tasks/build_utils.rb

@@ -1,4 +1,7 @@
 # frozen_string_literal: true
+require 'erb'
+require 'dotenv'
+require 'tempfile'
 
 module BuildUtils
   def proj_root
@@ -69,4 +72,109 @@ def image_name
   def user
     @user ||= Etc.getpwnam(Etc.getlogin)
   end
+
+  def os_release
+    @os_release ||= begin
+      if File.exist?('/etc/os-release')
+        Dotenv.parse('/etc/os-release')
+      else
+        {}
+      end
+    end
+  end
+
+  def scl_apache?
+    return true if (el? && os_release['VERSION_ID'] =~ /^7/)
+    false
+  end
+
+  def el?
+    return true if "#{os_release['ID']} #{os_release['ID_LIKE']}" =~ /(rhel|fedora)/
+    false
+  end
+
+  def debian?
+    return true if (os_release['ID'] =~ /(ubuntu|debian)/ or os_release['ID_LIKE'] == 'debian')
+    false
+  end
+
+  def apache_daemon
+    return '/opt/rh/httpd24/root/usr/sbin/httpd-scl-wrapper' if scl_apache?
+    "/usr/sbin/#{apache_service}"
+  end
+
+  def apache_reload
+    return '/usr/sbin/apachectl graceful' if debian?
+    "#{apache_daemon} $OPTIONS -k graceful"
+  end
+
+  def apache_user
+    return 'www-data' if debian?
+    'apache'
+  end
+
+  def apache_service
+    return 'apache2' if debian?
+    return 'httpd24-httpd' if scl_apache?
+    'httpd'
+  end
+
+  def infrastructure_files
+    @infrastructure_files ||= [
+      {
+        src: 'apache-systemd.ood-portal.conf.erb',
+        dest: File.join(DESTDIR, "etc/systemd/system/#{apache_service}.service.d/ood-portal.conf"),
+        mode: 0444,
+      },
+      {
+        src: 'apache-systemd.ood.conf',
+        dest: File.join(DESTDIR, "etc/systemd/system/#{apache_service}.service.d/ood.conf"),
+        mode: 0444,
+      },
+      {
+        src: 'crontab',
+        dest: File.join(DESTDIR, 'etc/cron.d/ood'),
+        mode: 0644,
+      },
+      {
+        src: 'favicon.ico',
+        dest: File.join(DESTDIR, 'var/www/ood/public/favicon.ico'),
+        mode: 0644,
+      },
+      {
+        src: 'logo.png',
+        dest: File.join(DESTDIR, 'var/www/ood/public/logo.png'),
+        mode: 0644,
+      },
+      {
+        src: 'logrotate',
+        dest: File.join(DESTDIR, 'etc/logrotate.d/ood'),
+        mode: 0644,
+      },
+      {
+        src: 'ondemand-nginx-tmpfiles',
+        dest: File.join(DESTDIR, 'usr/lib/tmpfiles.d/ondemand-nginx.conf'),
+        mode: 0644,
+      },
+      {
+        src: 'sudo.erb',
+        dest: File.join(DESTDIR, 'etc/sudoers.d/ood'),
+        mode: 0440,
+      },
+    ]
+  end
+
+  def render_package_file(name)
+    src = File.join(proj_root, 'packaging/files', name)
+    return src unless File.extname(name) == '.erb'
+
+    content = ERB.new(File.read(src), nil, '-').result(binding)
+    begin
+      t = Tempfile.new(name)
+      t.write(content)
+      t.path
+    ensure
+      t.close
+    end
+  end
 end

lib/tasks/packaging.rb

@@ -4,6 +4,10 @@
   require_relative 'build_utils'
   include BuildUtils
 
+  task :version do
+    puts ood_package_version
+  end
+
   def image_exists?(image_name)
     `#{container_runtime} inspect --type image --format exists #{image_name} || true`.chomp.eql?('exists')
   end

nginx_stage/etc/profile

@@ -8,4 +8,9 @@
 #
 SCL_PKGS=${SCL_PKGS:-"ondemand"}
 SCL_SOURCE="$(command -v scl_source)"
-[[ "${SCL_SOURCE}" ]] && source "${SCL_SOURCE}" enable ${SCL_PKGS}
+DEB_SOURCE="/opt/ood/ondemand/enable"
+if [[ "${SCL_SOURCE}" ]]; then
+  source "${SCL_SOURCE}" enable ${SCL_PKGS}
+else
+  [[ -e "${DEB_SOURCE}" ]] && source "${DEB_SOURCE}" || :
+fi

nginx_stage/lib/nginx_stage/configuration.rb

@@ -1,6 +1,7 @@
 require 'yaml'
 require 'etc'
 require 'pathname'
+require 'dotenv'
 
 module NginxStage
   # An object that stores the configuration options to control NginxStage's
@@ -419,7 +420,11 @@ def set_default_configuration
       self.ondemand_title        = nil
       self.template_root         = "#{root}/templates"
 
-      self.proxy_user       = 'apache'
+      if debian?
+        self.proxy_user = 'www-data'
+      else
+        self.proxy_user = 'apache'
+      end
       self.nginx_bin        = '/opt/ood/ondemand/root/usr/sbin/nginx'
       self.nginx_signals    = %i(stop quit reopen reload)
       self.mime_types_path  = '/opt/ood/ondemand/root/etc/nginx/mime.types'
@@ -508,6 +513,12 @@ def read_configuration(file)
       end
     end
 
+    def debian?
+      env = Dotenv.parse('/etc/os-release')
+      return true if (env['ID'] =~ /(ubuntu|debian)/ or env['ID_LIKE'] == 'debian')
+      false
+    end
+
     private
       # Recursively symbolize keys in hash
       def symbolize(obj)

nginx_stage/nginx_stage.gemspec

@@ -18,6 +18,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "syslog", "~> 0.1.0"
+  spec.add_dependency 'dotenv', '~> 2.1'
 
   spec.add_development_dependency "bundler", "~> 2.1.0"
   spec.add_development_dependency "rake", "~> 13.0.1"