CSV versioning (#1016)

* pkg/scaffold/olm-catalog: CSV directory and file versioning
* commands/.../olm-catalog: `--from-version` to specify which existing CSV to use as a base for the new CSV with version `--csv-version`; `--update-crds` to direct the SDK to copy CRD's from `deploy/crds` when creating a new CSV or updating an existing CSV

Author: estroz

commands/operator-sdk/cmd/olm-catalog/gen-csv.go

@@ -16,8 +16,11 @@ package catalog
 
 import (
 	"fmt"
+	"io/ioutil"
 	"path/filepath"
+	"strings"
 
+	"github.com/operator-framework/operator-sdk/internal/util/fileutil"
 	"github.com/operator-framework/operator-sdk/internal/util/projutil"
 	"github.com/operator-framework/operator-sdk/pkg/scaffold"
 	"github.com/operator-framework/operator-sdk/pkg/scaffold/input"
@@ -30,7 +33,9 @@ import (
 
 var (
 	csvVersion    string
+	fromVersion   string
 	csvConfigPath string
+	updateCRDs    bool
 )
 
 func NewGenCSVCmd() *cobra.Command {
@@ -40,15 +45,19 @@ func NewGenCSVCmd() *cobra.Command {
 		Long: `The gen-csv command generates a Cluster Service Version (CSV) YAML manifest
 for the operator. This file is used to publish the operator to the OLM Catalog.
 
-A CSV semantic version is supplied via the --csv-version flag.
+A CSV semantic version is supplied via the --csv-version flag. If your operator
+has already generated a CSV manifest you want to use as a base, supply its
+version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.
 
 Configure CSV generation by writing a config file 'deploy/olm-catalog/csv-config.yaml`,
 		RunE: genCSVFunc,
 	}
 
 	genCSVCmd.Flags().StringVar(&csvVersion, "csv-version", "", "Semantic version of the CSV")
 	genCSVCmd.MarkFlagRequired("csv-version")
+	genCSVCmd.Flags().StringVar(&fromVersion, "from-version", "", "Semantic version of an existing CSV to use as a base")
 	genCSVCmd.Flags().StringVar(&csvConfigPath, "csv-config", "", "Path to CSV config file. Defaults to deploy/olm-catalog/csv-config.yaml")
+	genCSVCmd.Flags().BoolVar(&updateCRDs, "update-crds", false, "Update CRD manifests in deploy/{operator-name}/{csv-version} the using latest API's")
 
 	return genCSVCmd
 }
@@ -74,25 +83,73 @@ func genCSVFunc(cmd *cobra.Command, args []string) error {
 	log.Infof("Generating CSV manifest version %s", csvVersion)
 
 	s := &scaffold.Scaffold{}
-	err := s.Execute(cfg,
-		&catalog.CSV{CSVVersion: csvVersion, ConfigFilePath: csvConfigPath},
-		&catalog.ConcatCRD{ConfigFilePath: csvConfigPath},
-	)
-	if err != nil {
+	csv := &catalog.CSV{
+		CSVVersion:     csvVersion,
+		FromVersion:    fromVersion,
+		ConfigFilePath: csvConfigPath,
+	}
+	if err := s.Execute(cfg, csv); err != nil {
 		return fmt.Errorf("catalog scaffold failed: (%v)", err)
 	}
+
+	// Write CRD's to the new or updated CSV package dir.
+	if updateCRDs {
+		input, err := csv.GetInput()
+		if err != nil {
+			return err
+		}
+		cfg, err := catalog.GetCSVConfig(csvConfigPath)
+		if err != nil {
+			return err
+		}
+		err = writeCRDsToDir(cfg.CRDCRPaths, filepath.Dir(input.Path))
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
 func verifyGenCSVFlags() error {
-	v, err := semver.NewVersion(csvVersion)
+	if err := verifyCSVVersion(csvVersion); err != nil {
+		return err
+	}
+	if fromVersion != "" {
+		if err := verifyCSVVersion(fromVersion); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func verifyCSVVersion(version string) error {
+	v, err := semver.NewVersion(version)
 	if err != nil {
-		return fmt.Errorf("%s is not a valid semantic version: (%v)", csvVersion, err)
+		return fmt.Errorf("%s is not a valid semantic version: (%v)", version, err)
 	}
 	// Ensures numerical values composing csvVersion don't contain leading 0's,
 	// ex. 01.01.01
-	if v.String() != csvVersion {
-		return fmt.Errorf("provided CSV version %s contains bad values (parses to %s)", csvVersion, v)
+	if v.String() != version {
+		return fmt.Errorf("provided CSV version %s contains bad values (parses to %s)", version, v)
+	}
+	return nil
+}
+
+func writeCRDsToDir(crdPaths []string, toDir string) error {
+	for _, p := range crdPaths {
+		if !strings.HasSuffix(p, "crd.yaml") {
+			continue
+		}
+		b, err := ioutil.ReadFile(p)
+		if err != nil {
+			return err
+		}
+		path := filepath.Join(toDir, filepath.Base(p))
+		err = ioutil.WriteFile(path, b, fileutil.DefaultFileMode)
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }

doc/sdk-cli-reference.md

@@ -178,25 +178,26 @@ Parent command for all OLM Catalog related commands.
 
 ### gen-csv
 
-Writes a Cluster Service Version (CSV) manifest and concatenated CRD files to `deploy/olm-catalog`.
+Writes a Cluster Service Version (CSV) manifest and optionally CRD files to `deploy/olm-catalog/{operator-name}/{csv-version}`.
 
 #### Flags
 
-* `--csv-version` (required) Semantic version of the CSV manifest.
-* `--csv-config` Path to CSV config file. Defaults to deploy/olm-catalog/csv-config.yaml.
+* `--csv-version` string - (required) Semantic version of the CSV manifest.
+* `--from-version` string - Semantic version of CSV manifest to use as a base for a new version.
+* `--csv-config` string - Path to CSV config file. Defaults to deploy/olm-catalog/csv-config.yaml.
+* `--update-crds` Update CRD manifests in deploy/{operator-name}/{csv-version} using the latest CRD manifests.
 
 #### Example
 
 ```console
-$ operator-sdk olm-catalog gen-csv --csv-version 0.1.1
-INFO[0000] Generating CSV manifest version 0.1.1
-INFO[0000] Fill in the following required fields in file deploy/olm-catalog/operator-name.csv.yaml:
+$ operator-sdk olm-catalog gen-csv --csv-version 0.1.0 --update-crds
+INFO[0000] Generating CSV manifest version 0.1.0
+INFO[0000] Fill in the following required fields in file deploy/olm-catalog/operator-name/0.1.0/operator-name.v0.1.0.clusterserviceversion.yaml:
 	spec.keywords
 	spec.maintainers
 	spec.provider
 	spec.labels
-INFO[0000] Created deploy/olm-catalog/operator-name.csv.yaml     
-INFO[0000] Created deploy/olm-catalog/_generated.concat_crd.yaml
+INFO[0000] Created deploy/olm-catalog/operator-name/0.1.0/operator-name.v0.1.0.clusterserviceversion.yaml     
 ```
 
 ## migrate
@@ -385,7 +386,7 @@ Run scorecard tests on an operator
 ### Example
 
 ```console
-$ operator-sdk scorecard --cr-manifest deploy/crds/cache_v1alpha1_memcached_cr.yaml --csv-path deploy/memcachedoperator.0.0.2.csv.yaml
+$ operator-sdk scorecard --cr-manifest deploy/crds/cache_v1alpha1_memcached_cr.yaml --csv-path deploy/olm-catalog/memcached-operator/0.0.2/memcached-operator.v0.0.2.clusterserviceversion.yaml
 Checking for existence of spec and status blocks in CR
 Checking that operator actions are reflected in status
 Checking that writing into CRs has an effect

doc/test-framework/scorecard.md

@@ -41,12 +41,12 @@ $ operator-sdk scorecard --cr-manifest deploy/crds/app_operator_cr.yaml --csv-pa
 The scorecard supports the use of a config file instead of or in addition to flags for configuration. By default, the scorecard will look
 for a file called `.osdk-scorecard` with either a `.yaml`, `.json`, or `.toml` file extension. You can also
 specify a different config file with the `--config` flag. The configuration options in the config file match the flags.
-For instance, for the flags `--cr-manifest "deploy/crds/cache_v1alpha1_memcached_cr.yaml" --init-timeout 60 --csv-path "deploy/memcachedoperator.0.0.2.csv.yaml"`, the corresponding yaml config file would contain:
+For instance, for the flags `--cr-manifest "deploy/crds/cache_v1alpha1_memcached_cr.yaml" --init-timeout 60 --csv-path "deploy/olm-catalog/memcached-operator/0.0.2/memcached-operator.v0.0.2.clusterserviceversion.yaml"`, the corresponding yaml config file would contain:
 
 ```yaml
 cr-manifest: "deploy/crds/cache_v1alpha1_memcached_cr.yaml"
 init-timeout: 60
-csv-path: "deploy/memcachedoperator.0.0.2.csv.yaml"
+csv-path: "deploy/olm-catalog/memcached-operator/0.0.2/memcached-operator.v0.0.2.clusterserviceversion.yaml"
 ```
 
 The hierarchy of config methods from highest priority to least is: flag->file->default.

hack/tests/scorecard-subcommand.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 DEST_IMAGE="quay.io/example/scorecard-proxy"
+CSV_PATH="deploy/olm-catalog/memcached-operator/0.0.2/memcached-operator.v0.0.2.clusterserviceversion.yaml"
 CONFIG_PATH=".test-osdk-scorecard.yaml"
 
 set -ex
@@ -13,7 +14,7 @@ pushd test/test-framework
 commandoutput="$(operator-sdk scorecard \
   --cr-manifest deploy/crds/cache_v1alpha1_memcached_cr.yaml \
   --init-timeout 60 \
-  --csv-path deploy/memcachedoperator.0.0.2.csv.yaml \
+  --csv-path "$CSV_PATH" \
   --verbose \
   --proxy-image "$DEST_IMAGE" \
   --proxy-pull-policy Never \

pkg/scaffold/olm-catalog/concat_crd.go

@@ -37,7 +37,7 @@ type CSVConfig struct {
 }
 
 // TODO: discuss case of no config file at default path: write new file or not.
-func getCSVConfig(cfgFile string) (*CSVConfig, error) {
+func GetCSVConfig(cfgFile string) (*CSVConfig, error) {
 	cfg := &CSVConfig{}
 	if _, err := os.Stat(cfgFile); err == nil {
 		cfgData, err := ioutil.ReadFile(cfgFile)