Make it possible to select a subpath of cloned directory

History.md

@@ -2,8 +2,12 @@
 
 ## New and Noteworthy
 
+*   Support for Mercurial is removed. The only SCM supported in the project-list.xml is "git".
+
 ## Fixed Issues
 
+*   [#89](https://github.com/pmd/pmd-regression-tester/pull/89): Make it possible to select a subpath of cloned directory
+
 ## External Contributions
 
 # 1.1.2 / 2021-04-20

Manifest.txt

@@ -20,6 +20,7 @@ config/design.xml
 config/project-list.xml
 config/projectlist_1_0_0.xsd
 config/projectlist_1_1_0.xsd
+config/projectlist_1_2_0.xsd
 lib/pmdtester.rb
 lib/pmdtester/builders/liquid_renderer.rb
 lib/pmdtester/builders/pmd_report_builder.rb

config/project-list.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 
 <projectlist xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-xsi:noNamespaceSchemaLocation="projectlist_1_0_0.xsd">
+xsi:noNamespaceSchemaLocation="projectlist_1_2_0.xsd">
   <description>Standard Projects</description>
 
   <project>
@@ -20,10 +20,11 @@ xsi:noNamespaceSchemaLocation="projectlist_1_0_0.xsd">
     <tag>v5.0.6.RELEASE</tag>
   </project>
 
-  <!---<project>
-    <name>openjdk10</name>
-    <type>hg</type>
-    <connection>http://hg.openjdk.java.net/jdk10/jdk10/jdk</connection>
-    <webview-url>http://hg.openjdk.java.net/jdk10/jdk10/jdk/file/777356696811</webview-url>
-  </project> -->
+  <project>
+    <name>openjdk-11</name>
+    <type>git</type>
+    <connection>https://github.com/openjdk/jdk</connection>
+    <tag>jdk-11+28</tag>
+    <src-subpath>src/java.base</src-subpath>
+  </project>
 </projectlist>

config/projectlist_1_2_0.xsd

@@ -0,0 +1,39 @@
+<?xml version="1.0" ?>
+<!-- version 1.1.0 -->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
+  <xs:element name="projectlist">
+    <xs:complexType>
+      <xs:sequence>
+        <xs:element name="description" type="xs:string" minOccurs="1" maxOccurs="1"/>
+        <xs:element name="project" type="project" minOccurs="1" maxOccurs="unbounded"/>
+      </xs:sequence>
+    </xs:complexType>
+  </xs:element>
+  <xs:complexType name="project">
+    <xs:sequence>
+      <xs:element name="name" type="xs:string" minOccurs="1" maxOccurs="1"/>
+      <xs:element name="type" minOccurs="1" maxOccurs="1">
+        <xs:simpleType>
+          <xs:restriction base="xs:string">
+            <xs:enumeration value="git"/>
+          </xs:restriction>
+        </xs:simpleType>
+      </xs:element>
+      <xs:element name="connection" type="xs:string" minOccurs="1" maxOccurs="1"/>
+      <xs:element name="webview-url" type="xs:string" minOccurs="0" maxOccurs="1"/>
+      <xs:element name="tag" type="xs:string" minOccurs="0" maxOccurs="1"/>
+      <xs:element name="src-subpath" type="xs:string" minOccurs="0" maxOccurs="1" default=".">
+        <xs:annotation>
+          <xs:documentation>
+            Value of the -dir option for the PMD run.
+            The value must be a directory path relative to the root directory of the clone.
+            Defaults to just '.', ie the entire directory.
+          </xs:documentation>
+        </xs:annotation>
+      </xs:element>
+      <xs:element name="exclude-pattern" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+      <xs:element name="build-command" type="xs:string" minOccurs="0" maxOccurs="1"/>
+      <xs:element name="auxclasspath-command" type="xs:string" minOccurs="0" maxOccurs="1"/>
+    </xs:sequence>
+  </xs:complexType>
+</xs:schema>

lib/pmdtester/builders/pmd_report_builder.rb

@@ -69,7 +69,7 @@ def build_pmd(into_dir:)
                       ' -Dmaven.source.skip=true' \
                       ' -Dcheckstyle.skip=true' \
                       ' -Dpmd.skip=true' \
-                      ' -T1C'
+                      ' -T1C -B'
         Cmd.execute(package_cmd)
       end
 
@@ -119,7 +119,7 @@ def generate_config_for(project)
       doc = Nokogiri::XML(File.read(@branch_config))
       ruleset = doc.at_css('ruleset')
       ruleset.add_child("\n")
-      project.exclude_pattern.each do |exclude_pattern|
+      project.exclude_patterns.each do |exclude_pattern|
         ruleset.add_child("    <exclude-pattern>#{exclude_pattern}</exclude-pattern>\n")
       end
 

lib/pmdtester/builders/project_builder.rb

@@ -17,11 +17,19 @@ def clone_projects
 
       @projects.each do |project|
         logger.info "Start cloning #{project.name} repository"
-        path = project.local_source_path
-        clone_cmd = "#{project.type} clone #{project.connection} #{path}"
+        path = project.clone_root_path
+
         if File.exist?(path)
           logger.warn "Skipping clone, project path #{path} already exists"
         else
+          raise "Unsupported project type '#{project.type}' - only git is supported" unless project.type == 'git'
+
+          # git:
+          # Don't download whole history
+          # Note we don't use --single-branch, because the repo is downloaded
+          # once but may be used with several tags.
+          clone_cmd = "git clone --no-single-branch --depth 1 #{project.connection} #{path}"
+
           Cmd.execute(clone_cmd)
         end
 
@@ -38,7 +46,7 @@ def build_projects
       logger.info 'Building projects started'
 
       @projects.each do |project|
-        path = project.local_source_path
+        path = project.clone_root_path
         Dir.chdir(path) do
           progress_logger = SimpleProgressLogger.new("building #{project.name} in #{path}")
           progress_logger.start
@@ -87,12 +95,9 @@ def run_as_script(path, command)
     end
 
     def execute_reset_cmd(type, tag)
-      case type
-      when 'git'
-        reset_cmd = "git checkout #{tag}; git reset --hard #{tag}"
-      when 'hg'
-        reset_cmd = "hg up #{tag}"
-      end
+      raise "Unsupported project type '#{type}' - only git is supported" unless type == 'git'
+
+      reset_cmd = "git checkout #{tag}; git reset --hard #{tag}"
 
       Cmd.execute(reset_cmd)
     end

lib/pmdtester/parsers/projects_parser.rb

@@ -21,7 +21,7 @@ def parse(list_file)
     end
 
     def schema_file_path
-      ResourceLocator.locate('config/projectlist_1_1_0.xsd')
+      ResourceLocator.locate('config/projectlist_1_2_0.xsd')
     end
   end
 

lib/pmdtester/project.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'pathname'
+
 module PmdTester
   # This class represents all the information about the project
   class Project
@@ -12,15 +14,16 @@ class Project
     attr_reader :connection
     attr_reader :webview_url
     attr_reader :tag
-    attr_reader :exclude_pattern
+    attr_reader :exclude_patterns
+    attr_reader :src_subpath
     attr_accessor :report_diff
     # key: pmd branch name as String => value: local path of pmd report
     attr_reader :build_command
     attr_reader :auxclasspath_command
     # stores the auxclasspath calculated after cloning/preparing the project
     attr_accessor :auxclasspath
 
-    def initialize(project)
+    def initialize(project) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
       @name = project.at_xpath('name').text
       @type = project.at_xpath('type').text
       @connection = project.at_xpath('connection').text
@@ -31,9 +34,10 @@ def initialize(project)
       @webview_url = default_webview_url
       @webview_url = webview_url_element.text unless webview_url_element.nil?
 
-      @exclude_pattern = []
+      @src_subpath = project.at_xpath('src-subpath')&.text || '.'
+      @exclude_patterns = []
       project.xpath('exclude-pattern').each do |ep|
-        @exclude_pattern.push(ep.text)
+        @exclude_patterns.push(ep.text)
       end
 
       @build_command = project.at_xpath('build-command')&.text
@@ -56,17 +60,17 @@ def default_webview_url
     # Change the file path from 'LOCAL_DIR/SOURCE_CODE_PATH' to
     # 'WEB_VIEW_URL/SOURCE_CODE_PATH'
     def get_webview_url(file_path)
-      file_path.gsub(%r{/#{local_source_path}}, @webview_url)
+      file_path.gsub(%r{/#{clone_root_path}}, @webview_url)
     end
 
     # Change the file path from 'LOCAL_DIR/SOURCE_CODE_PATH' to
     # 'PROJECT_NAME/SOURCE_CODE_PATH'
     def get_path_inside_project(file_path)
-      file_path.gsub(%r{/#{local_source_path}}, @name)
+      file_path.gsub(%r{/#{clone_root_path}}, @name)
     end
 
     def get_local_path(file_path)
-      file_path.sub(%r{/#{local_source_path}/}, '')
+      file_path.sub(%r{/#{clone_root_path}/}, '')
     end
 
     def get_pmd_report_path(branch_name)
@@ -93,17 +97,26 @@ def get_config_path(branch_name)
       end
     end
 
+    ##
+    # Path to the sources to analyze (below or equal to clone_root_path)
+    def local_source_path
+      # normalize path
+      Pathname.new("#{clone_root_path}/#{src_subpath}").cleanpath
+    end
+
+    ##
+    # Path to the clone directory
+    def clone_root_path
+      "#{REPOSITORIES_PATH}/#{@name}"
+    end
+
     def get_project_target_dir(branch_name)
       branch_filename = PmdBranchDetail.branch_filename(branch_name)
       dir = "target/reports/#{branch_filename}/#{@name}"
       FileUtils.mkdir_p(dir) unless File.directory?(dir)
       dir
     end
 
-    def local_source_path
-      "#{REPOSITORIES_PATH}/#{@name}"
-    end
-
     def compute_report_diff(base_branch, patch_branch, filter_set)
       self.report_diff = build_report_diff(get_pmd_report_path(base_branch),
                                            get_pmd_report_path(patch_branch),

pmdtester.gemspec

@@ -1,22 +1,22 @@
 # DO NOT EDIT THIS FILE. Instead, edit Rakefile, and run `rake hoe:spec`.
 
 # -*- encoding: utf-8 -*-
-# stub: pmdtester 1.1.2 ruby lib
+# stub: pmdtester 1.2.0.pre.SNAPSHOT ruby lib
 
 Gem::Specification.new do |s|
   s.name = "pmdtester".freeze
-  s.version = "1.1.2"
+  s.version = "1.2.0.pre.SNAPSHOT"
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
+  s.required_rubygems_version = Gem::Requirement.new("> 1.3.1".freeze) if s.respond_to? :required_rubygems_version=
   s.metadata = { "bug_tracker_uri" => "https://github.com/pmd/pmd-regression-tester/issues", "homepage_uri" => "https://pmd.github.io", "source_code_uri" => "https://github.com/pmd/pmd-regression-tester" } if s.respond_to? :metadata=
   s.require_paths = ["lib".freeze]
   s.authors = ["Andreas Dangel".freeze, "Binguo Bao".freeze, "Cl\u00E9ment Fournier".freeze]
-  s.date = "2021-04-20"
+  s.date = "2021-06-17"
   s.description = "A regression testing tool ensure that new problems and unexpected behaviors will not be introduced to PMD project after fixing an issue , and new rules can work as expected.".freeze
   s.email = ["andreas.dangel@pmd-code.org".freeze, "djydewang@gmail.com".freeze, "clement.fournier76@gmail.com".freeze]
   s.executables = ["pmdtester".freeze]
   s.extra_rdoc_files = ["History.md".freeze, "Manifest.txt".freeze, "README.rdoc".freeze]
-  s.files = [".ci/build.sh".freeze, ".ci/inc/fetch_ci_scripts.bash".freeze, ".ci/manual-integration-tests.sh".freeze, ".github/workflows/build.yml".freeze, ".github/workflows/manual-integration-tests.yml".freeze, ".gitignore".freeze, ".hoerc".freeze, ".rubocop.yml".freeze, ".rubocop_todo.yml".freeze, ".ruby-version".freeze, "Gemfile".freeze, "History.md".freeze, "LICENSE".freeze, "Manifest.txt".freeze, "README.rdoc".freeze, "Rakefile".freeze, "bin/pmdtester".freeze, "config/all-java.xml".freeze, "config/design.xml".freeze, "config/project-list.xml".freeze, "config/projectlist_1_0_0.xsd".freeze, "config/projectlist_1_1_0.xsd".freeze, "lib/pmdtester.rb".freeze, "lib/pmdtester/builders/liquid_renderer.rb".freeze, "lib/pmdtester/builders/pmd_report_builder.rb".freeze, "lib/pmdtester/builders/project_builder.rb".freeze, "lib/pmdtester/builders/project_hasher.rb".freeze, "lib/pmdtester/builders/rule_set_builder.rb".freeze, "lib/pmdtester/builders/simple_progress_logger.rb".freeze, "lib/pmdtester/builders/summary_report_builder.rb".freeze, "lib/pmdtester/cmd.rb".freeze, "lib/pmdtester/collection_by_file.rb".freeze, "lib/pmdtester/parsers/options.rb".freeze, "lib/pmdtester/parsers/pmd_report_document.rb".freeze, "lib/pmdtester/parsers/projects_parser.rb".freeze, "lib/pmdtester/pmd_branch_detail.rb".freeze, "lib/pmdtester/pmd_configerror.rb".freeze, "lib/pmdtester/pmd_error.rb".freeze, "lib/pmdtester/pmd_report_detail.rb".freeze, "lib/pmdtester/pmd_tester_utils.rb".freeze, "lib/pmdtester/pmd_violation.rb".freeze, "lib/pmdtester/project.rb".freeze, "lib/pmdtester/report_diff.rb".freeze, "lib/pmdtester/resource_locator.rb".freeze, "lib/pmdtester/runner.rb".freeze, "pmdtester.gemspec".freeze, "resources/_includes/diff_pill_row.html".freeze, "resources/css/bootstrap.min.css".freeze, "resources/css/datatables.min.css".freeze, "resources/css/pmd-tester.css".freeze, "resources/js/bootstrap.min.js".freeze, "resources/js/code-snippets.js".freeze, "resources/js/datatables.min.js".freeze, "resources/js/jquery-3.2.1.slim.min.js".freeze, "resources/js/jquery.min.js".freeze, "resources/js/popper.min.js".freeze, "resources/js/project-report.js".freeze, "resources/project_diff_report.html".freeze, "resources/project_index.html".freeze]
+  s.files = [".ci/build.sh".freeze, ".ci/inc/fetch_ci_scripts.bash".freeze, ".ci/manual-integration-tests.sh".freeze, ".github/workflows/build.yml".freeze, ".github/workflows/manual-integration-tests.yml".freeze, ".gitignore".freeze, ".hoerc".freeze, ".rubocop.yml".freeze, ".rubocop_todo.yml".freeze, ".ruby-version".freeze, "Gemfile".freeze, "History.md".freeze, "LICENSE".freeze, "Manifest.txt".freeze, "README.rdoc".freeze, "Rakefile".freeze, "bin/pmdtester".freeze, "config/all-java.xml".freeze, "config/design.xml".freeze, "config/project-list.xml".freeze, "config/projectlist_1_0_0.xsd".freeze, "config/projectlist_1_1_0.xsd".freeze, "config/projectlist_1_2_0.xsd".freeze, "lib/pmdtester.rb".freeze, "lib/pmdtester/builders/liquid_renderer.rb".freeze, "lib/pmdtester/builders/pmd_report_builder.rb".freeze, "lib/pmdtester/builders/project_builder.rb".freeze, "lib/pmdtester/builders/project_hasher.rb".freeze, "lib/pmdtester/builders/rule_set_builder.rb".freeze, "lib/pmdtester/builders/simple_progress_logger.rb".freeze, "lib/pmdtester/builders/summary_report_builder.rb".freeze, "lib/pmdtester/cmd.rb".freeze, "lib/pmdtester/collection_by_file.rb".freeze, "lib/pmdtester/parsers/options.rb".freeze, "lib/pmdtester/parsers/pmd_report_document.rb".freeze, "lib/pmdtester/parsers/projects_parser.rb".freeze, "lib/pmdtester/pmd_branch_detail.rb".freeze, "lib/pmdtester/pmd_configerror.rb".freeze, "lib/pmdtester/pmd_error.rb".freeze, "lib/pmdtester/pmd_report_detail.rb".freeze, "lib/pmdtester/pmd_tester_utils.rb".freeze, "lib/pmdtester/pmd_violation.rb".freeze, "lib/pmdtester/project.rb".freeze, "lib/pmdtester/report_diff.rb".freeze, "lib/pmdtester/resource_locator.rb".freeze, "lib/pmdtester/runner.rb".freeze, "pmdtester.gemspec".freeze, "resources/_includes/diff_pill_row.html".freeze, "resources/css/bootstrap.min.css".freeze, "resources/css/datatables.min.css".freeze, "resources/css/pmd-tester.css".freeze, "resources/js/bootstrap.min.js".freeze, "resources/js/code-snippets.js".freeze, "resources/js/datatables.min.js".freeze, "resources/js/jquery-3.2.1.slim.min.js".freeze, "resources/js/jquery.min.js".freeze, "resources/js/popper.min.js".freeze, "resources/js/project-report.js".freeze, "resources/project_diff_report.html".freeze, "resources/project_index.html".freeze]
   s.homepage = "https://pmd.github.io".freeze
   s.licenses = ["BSD-2-Clause".freeze]
   s.rdoc_options = ["--main".freeze, "README.rdoc".freeze]

test/integration_test_pmd_report_builder.rb

@@ -11,7 +11,8 @@ def setup
 
   def test_build
     argv = %w[-r target/repositories/pmd -b master -p origin/pmd/7.0.x
-              -c test/resources/pmd7-config.xml -l test/resources/project-test.xml
+              -c test/resources/integration_test_pmd_report_builder/pmd7-config.xml
+              -l test/resources/integration_test_pmd_report_builder/project-test.xml
               --error-recovery --debug]
     options = PmdTester::Options.new(argv)
     projects = ProjectsParser.new.parse(options.project_list)

test/integration_test_runner.rb

@@ -9,7 +9,7 @@ def setup
 
   def test_local_mode
     argv = '-r target/repositories/pmd -b pmd_releases/6.7.0 -bc config/design.xml' \
-              ' -p master -pc config/design.xml -l test/resources/project-test.xml'
+              ' -p master -pc config/design.xml -l test/resources/integration_test_runner/project-test.xml'
 
     system("bundle exec bin/pmdtester #{argv}")
 

test/resources/html_report_builder/expected_empty_diff_report.html

@@ -2,7 +2,7 @@
 <html lang="en">
 <head>
     <meta charset="utf-8">
-    <title>Diff report for openjdk10</title>
+    <title>Diff report for openjdk-11</title>
 
     <link rel="stylesheet" type="text/css" href="../css/bootstrap.min.css"/>
     <link rel="stylesheet" type="text/css" href="../css/datatables.min.css"/>

test/resources/project-test.xml → test/resources/integration_test_pmd_report_builder/project-test.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 
 <projectlist xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-xsi:noNamespaceSchemaLocation="projectlist_1_1_0.xsd">
+xsi:noNamespaceSchemaLocation="projectlist_1_2_0.xsd">
   <description>Standard Projects</description>
 
   <project>
@@ -19,8 +19,8 @@ fi
 
 set -e
 
-mvn clean test-compile
-mvn dependency:build-classpath -DincludeScope=test -Dmdep.outputFile=classpath.txt
+mvn clean test-compile -B
+mvn dependency:build-classpath -DincludeScope=test -Dmdep.outputFile=classpath.txt -B
 ]]></build-command>
     <auxclasspath-command>echo -n "$(pwd)/target/classes:$(pwd)/target/test-classes:"; cat classpath.txt</auxclasspath-command>
   </project>

test/resources/integration_test_runner/project-list-single.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 
 <projectlist xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-xsi:noNamespaceSchemaLocation="projectlist_1_1_0.xsd">
+xsi:noNamespaceSchemaLocation="projectlist_1_2_0.xsd">
   <description>Projects used in integration_test_runner.rb</description>
 
   <project>
@@ -19,8 +19,8 @@ fi
 
 set -e
 
-mvn clean test-compile
-mvn dependency:build-classpath -DincludeScope=test -Dmdep.outputFile=classpath.txt
+mvn clean test-compile -B
+mvn dependency:build-classpath -DincludeScope=test -Dmdep.outputFile=classpath.txt -B
 ]]></build-command>
     <auxclasspath-command>echo -n "$(pwd)/target/classes:$(pwd)/target/test-classes:"; cat classpath.txt</auxclasspath-command>
   </project>

test/resources/integration_test_runner/project-list.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 
 <projectlist xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-xsi:noNamespaceSchemaLocation="projectlist_1_0_0.xsd">
+xsi:noNamespaceSchemaLocation="projectlist_1_2_0.xsd">
   <description>Standard Projects</description>
 
   <project>
@@ -23,11 +23,4 @@ xsi:noNamespaceSchemaLocation="projectlist_1_0_0.xsd">
     <tag>v5.0.6.RELEASE</tag>
   </project>
 -->
-
-  <!---<project>
-    <name>openjdk10</name>
-    <type>hg</type>
-    <connection>http://hg.openjdk.java.net/jdk10/jdk10/jdk</connection>
-    <webview-url>http://hg.openjdk.java.net/jdk10/jdk10/jdk/file/777356696811</webview-url>
-  </project> -->
 </projectlist>