check for plugins file schema version and type (#2532)

* check for plugins file schema version and type

Signed-off-by: Yingrong Zhao <yingrong.zhao@gmail.com>

Author: VinozzZ

docs/content/blog/install-multiple-plugins.md

@@ -0,0 +1,48 @@
+
+---
+title: "Quickly set up a Porter environment with required plugins"
+description: "How to install multiple plugins with Porter"
+date: "2023-01-24"
+authorname: "Yingrong Zhao"
+author: "@vinozzz"
+authorlink: "https://github.com/vinozzz"
+authorimage: "https://github.com/vinozzz.png"
+tags: ["best-practice", "plugins"]
+summary: | 
+    Setting up your Porter environment with your required plugins using the new `--file` flag with `porter plugins install` command.
+---
+
+### Breaking change
+The recent porter v1.0.5 release introduced a new flag `--file` on `porter plugins install` command. Its intention is to allow users to install multiple plugins through a plugins definition file with a single porter command. However, it did not work as expected due to bad file format.
+
+The fix that contains the correct schema has been published with a new v1.0.6 release. If you have an existing plugins file, please update it to work with v1.0.6+.
+
+### Install multiple plugins with a single command
+Now, you can install multiple plugins using a plugin definition yaml file like below:
+```yaml
+schemaType: Plugins
+schemaVersion: 1.0.0
+plugins:
+  azure:
+    version: v1.0.1
+  kubernetes:
+    version: v1.0.1
+```
+
+After creating the file, you can run the command:
+```bash
+porter plugins install -f <path-to-the-file>
+```
+
+The output from the command should look like this:
+```
+installed azure plugin v1.0.1 (e361abc)
+installed kubernetes plugin v1.0.1 (f01c944)
+```
+
+Make sure to update your current plugins schema file to the [latest format](/reference/file-formats/#plugins) 
+Please [let us know][contact] how the change went (good or bad), and we are happy to help if you have questions, or you would like help with your migration.
+
+[announced]: https://github.com/docker/roadmap/issues/209
+[Install Porter]: /install/
+[contact]: /community/

docs/content/reference/file-formats.md

@@ -162,21 +162,22 @@ You can use this [json schema][plugins-schema] to validate a plugins config file
 ```yaml
 schemaType: Plugins
 schemaVersion: 1.0.0
-azure:
-  version: v1.0.0
-  feedURL: https://cdn.porter.sh/plugins/atom.xml
-  url: https://example.com
-  mirror: https://example.com
+plugins:
+  azure:
+    version: v1.0.0
+    feedURL: https://cdn.porter.sh/plugins/atom.xml
+    url: https://example.com
+    mirror: https://example.com
 ```
 
 | Field                | Required | Description                                                                                                                                    |
 |----------------------|----------|------------------------------------------------------------------------------------------------------------------------------------------------|
 | schemaType           | false    | The type of document. This isn't used by Porter but is included when Porter outputs the file, so that editors can determine the resource type. |
 | schemaVersion        | true     | The version of the Plugins schema used in this file.                                                                                     |
-| <pluginName>.version | false    | The version of the plugin.                                                                                                                 |
-| <pluginName>.feedURL | false    | The url of an atom feed where the plugin can be downloaded.
-| <pluginName>.url     | false    | The url from where the plugin can be downloaded.                                                                                                                 |
-| <pluginName>.mirror  | false    | The mirror of official Porter assets.                                                                                                                 |
+| plugins.<pluginName>.version | false    | The version of the plugin.                                                                                                                 |
+| plugins.<pluginName>.feedURL | false    | The url of an atom feed where the plugin can be downloaded.
+| plugins.<pluginName>.url     | false    | The url from where the plugin can be downloaded.                                                                                                                 |
+| plugins.<pluginName>.mirror  | false    | The mirror of official Porter assets.                                                                                                                 |
 
 [cs-schema]: /schema/v1/credential-set.schema.json
 [ps-schema]: /schema/v1/parameter-set.schema.json

pkg/plugins/install.go

@@ -3,11 +3,17 @@ package plugins
 import (
 	"fmt"
 	"sort"
+	"strings"
 
 	"get.porter.sh/porter/pkg/pkgmgmt"
 	"get.porter.sh/porter/pkg/portercontext"
+	"github.com/cnabio/cnab-go/schema"
 )
 
+// InstallPluginsSchemaVersion represents the version associated with the schema
+// plugins configuration documents.
+var InstallPluginsSchemaVersion = schema.Version("1.0.0")
+
 type InstallOptions struct {
 	pkgmgmt.InstallOptions
 
@@ -25,8 +31,9 @@ func (o *InstallOptions) Validate(args []string, cxt *portercontext.Context) err
 			return fmt.Errorf("plugin URL should not be specified when --file is provided")
 		}
 
-		if o.Version != "" {
-			return fmt.Errorf("plugin version should not be specified when --file is provided")
+		// version should not be set to anything other than the default value
+		if o.Version != "" && o.Version != "latest" {
+			return fmt.Errorf("plugin version %s should not be specified when --file is provided", o.Version)
 		}
 
 		if _, err := cxt.FileSystem.Stat(o.File); err != nil {
@@ -39,20 +46,41 @@ func (o *InstallOptions) Validate(args []string, cxt *portercontext.Context) err
 	return o.InstallOptions.Validate(args)
 }
 
-// InstallFileOption is the go representation of plugin installation file format.
-type InstallFileOption map[string]pkgmgmt.InstallOptions
+// InstallPluginsSpec represents the user-defined configuration for plugins installation.
+type InstallPluginsSpec struct {
+	SchemaType    string               `yaml:"schemaType"`
+	SchemaVersion string               `yaml:"schemaVersion"`
+	Plugins       InstallPluginsConfig `yaml:"plugins"`
+}
+
+func (spec InstallPluginsSpec) Validate() error {
+	if spec.SchemaType != "" && strings.ToLower(spec.SchemaType) != "plugins" {
+		return fmt.Errorf("invalid schemaType %s, expected Plugins", spec.SchemaType)
+	}
+
+	if InstallPluginsSchemaVersion != schema.Version(spec.SchemaVersion) {
+		if spec.SchemaVersion == "" {
+			spec.SchemaVersion = "(none)"
+		}
+		return fmt.Errorf("invalid schemaVersion provided: %s. This version of Porter is compatible with %s.", spec.SchemaVersion, InstallPluginsSchemaVersion)
+	}
+	return nil
+}
+
+// InstallPluginsConfig is the go representation of plugin installation file format.
+type InstallPluginsConfig map[string]pkgmgmt.InstallOptions
 
-// InstallPluginsConfig is a sorted list of InstallationFileOption in alphabetical order.
-type InstallPluginsConfig struct {
-	data InstallFileOption
+// InstallPluginsConfigList is a sorted list of InstallationFileOption in alphabetical order.
+type InstallPluginsConfigList struct {
+	data InstallPluginsConfig
 	keys []string
 }
 
 // NewInstallPluginConfigs returns a new instance of InstallPluginConfigs with plugins sorted in alphabetical order
 // using their names.
-func NewInstallPluginConfigs(opt InstallFileOption) InstallPluginsConfig {
+func NewInstallPluginConfigs(opt InstallPluginsConfig) InstallPluginsConfigList {
 	keys := make([]string, 0, len(opt))
-	data := make(InstallFileOption, len(opt))
+	data := make(InstallPluginsConfig, len(opt))
 	for k, v := range opt {
 		keys = append(keys, k)
 
@@ -65,14 +93,14 @@ func NewInstallPluginConfigs(opt InstallFileOption) InstallPluginsConfig {
 		return keys[i] < keys[j]
 	})
 
-	return InstallPluginsConfig{
+	return InstallPluginsConfigList{
 		data: data,
 		keys: keys,
 	}
 }
 
-// Configs returns InstallOptions list in alphabetical order.
-func (pc InstallPluginsConfig) Configs() []pkgmgmt.InstallOptions {
+// Values returns InstallOptions list in alphabetical order.
+func (pc InstallPluginsConfigList) Values() []pkgmgmt.InstallOptions {
 	value := make([]pkgmgmt.InstallOptions, 0, len(pc.keys))
 	for _, k := range pc.keys {
 		value = append(value, pc.data[k])

pkg/plugins/install_test.go

@@ -19,9 +19,9 @@ func TestInstallOptions_Validate(t *testing.T) {
 }
 
 func TestInstallPluginsConfig(t *testing.T) {
-	input := InstallFileOption{"kubernetes": pkgmgmt.InstallOptions{URL: "test-kubernetes.com"}, "azure": pkgmgmt.InstallOptions{URL: "test-azure.com"}}
+	input := InstallPluginsConfig{"kubernetes": pkgmgmt.InstallOptions{URL: "test-kubernetes.com"}, "azure": pkgmgmt.InstallOptions{URL: "test-azure.com"}}
 	expected := []pkgmgmt.InstallOptions{{Name: "azure", PackageType: "plugin", URL: "test-azure.com"}, {Name: "kubernetes", PackageType: "plugin", URL: "test-kubernetes.com"}}
 
 	cfg := NewInstallPluginConfigs(input)
-	require.Equal(t, expected, cfg.Configs())
+	require.Equal(t, expected, cfg.Values())
 }

pkg/porter/plugins.go

@@ -158,23 +158,23 @@ func (p *Porter) InstallPlugin(ctx context.Context, opts plugins.InstallOptions)
 	ctx, log := tracing.StartSpan(ctx)
 	defer log.EndSpan()
 
-	installConfigs, err := p.getPluginInstallConfigs(ctx, opts)
+	installOpts, err := p.getPluginInstallOptions(ctx, opts)
 	if err != nil {
 		return err
 	}
-	for _, cfg := range installConfigs {
-		err := p.Plugins.Install(ctx, cfg)
+	for _, opt := range installOpts {
+		err := p.Plugins.Install(ctx, opt)
 		if err != nil {
 			return err
 		}
 
-		plugin, err := p.Plugins.GetMetadata(ctx, cfg.Name)
+		plugin, err := p.Plugins.GetMetadata(ctx, opt.Name)
 		if err != nil {
 			return fmt.Errorf("failed to get plugin metadata: %w", err)
 		}
 
 		v := plugin.GetVersionInfo()
-		fmt.Fprintf(p.Out, "installed %s plugin %s (%s)\n", cfg.Name, v.Version, v.Commit)
+		fmt.Fprintf(p.Out, "installed %s plugin %s (%s)\n", opt.Name, v.Version, v.Commit)
 	}
 
 	return nil
@@ -191,13 +191,13 @@ func (p *Porter) UninstallPlugin(ctx context.Context, opts pkgmgmt.UninstallOpti
 	return nil
 }
 
-func (p *Porter) getPluginInstallConfigs(ctx context.Context, opts plugins.InstallOptions) ([]pkgmgmt.InstallOptions, error) {
+func (p *Porter) getPluginInstallOptions(ctx context.Context, opts plugins.InstallOptions) ([]pkgmgmt.InstallOptions, error) {
 	_, log := tracing.StartSpan(ctx)
 	defer log.EndSpan()
 
 	var installConfigs []pkgmgmt.InstallOptions
 	if opts.File != "" {
-		var data plugins.InstallFileOption
+		var data plugins.InstallPluginsSpec
 		if log.ShouldLog(zapcore.DebugLevel) {
 			// ignoring any error here, printing debug info isn't critical
 			contents, _ := p.FileSystem.ReadFile(opts.File)
@@ -207,9 +207,14 @@ func (p *Porter) getPluginInstallConfigs(ctx context.Context, opts plugins.Insta
 		if err := encoding.UnmarshalFile(p.FileSystem, opts.File, &data); err != nil {
 			return nil, fmt.Errorf("unable to parse %s as an installation document: %w", opts.File, err)
 		}
-		sortedCfgs := plugins.NewInstallPluginConfigs(data)
 
-		for _, config := range sortedCfgs.Configs() {
+		if err := data.Validate(); err != nil {
+			return nil, err
+		}
+
+		sortedCfgs := plugins.NewInstallPluginConfigs(data.Plugins)
+
+		for _, config := range sortedCfgs.Values() {
 			// if user specified a feed url or mirror using the flags, it will become
 			// the default value and apply to empty values parsed from the provided file
 			if config.FeedURL == "" {

pkg/porter/plugins_test.go

@@ -3,15 +3,21 @@ package porter
 import (
 	"context"
 	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
 	"testing"
 
 	"get.porter.sh/porter/pkg/pkgmgmt"
 	"get.porter.sh/porter/pkg/pkgmgmt/client"
 	"get.porter.sh/porter/pkg/plugins"
 	"get.porter.sh/porter/pkg/printer"
 	"get.porter.sh/porter/pkg/test"
+	"get.porter.sh/porter/pkg/yaml"
+	"get.porter.sh/porter/tests"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/xeipuuv/gojsonschema"
 )
 
 func TestPorter_PrintPlugins(t *testing.T) {
@@ -310,6 +316,63 @@ func TestPorter_InstallPlugin(t *testing.T) {
 	}
 }
 
+func TestPorter_InstallPluginsSchema(t *testing.T) {
+	p := NewTestPorter(t)
+	schema, err := os.ReadFile(filepath.Join(p.RepoRoot, "pkg/schema/plugins.schema.json"))
+	require.NoError(t, err, "failed to read plugins.schema.json file")
+	testcases := []struct {
+		name    string
+		path    string
+		wantErr string
+	}{
+		{
+			name:    "valid",
+			path:    "testdata/plugins.json",
+			wantErr: "",
+		},
+		{
+			name:    "invalid",
+			path:    "testdata/invalid-plugins.json",
+			wantErr: "(root): Additional property invalid-field is not allowed\nplugins.plugin1: Additional property random-field is not allowed\n",
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Load manifest as a go dump
+			testManifest, err := os.ReadFile(tc.path)
+			require.NoError(t, err, "failed to read %s", tc.path)
+
+			m := make(map[string]interface{})
+			err = yaml.Unmarshal(testManifest, &m)
+			require.NoError(t, err, "failed to unmarshal %s", tc.path)
+
+			// Load the manifest schema returned from `porter schema`
+			manifestLoader := gojsonschema.NewGoLoader(m)
+			schemaLoader := gojsonschema.NewBytesLoader(schema)
+
+			// Validate the manifest against the schema
+			fails, err := gojsonschema.Validate(schemaLoader, manifestLoader)
+			require.NoError(t, err)
+
+			if tc.wantErr == "" {
+				assert.Empty(t, fails.Errors(), "expected %s to validate against the plugins schema", tc.path)
+				// Print any validation errors returned
+				for _, err := range fails.Errors() {
+					t.Logf("%s", err)
+				}
+			} else {
+				var allFails strings.Builder
+				for _, err := range fails.Errors() {
+					allFails.WriteString(err.String())
+					allFails.WriteString("\n")
+				}
+				tests.RequireOutputContains(t, tc.wantErr, allFails.String())
+			}
+		})
+	}
+}
+
 func TestPorter_UninstallPlugin(t *testing.T) {
 	ctx := context.Background()
 	p := NewTestPorter(t)

pkg/porter/testdata/invalid-plugins.json

@@ -0,0 +1,13 @@
+{
+  "schemaType": "Plugins",
+  "schemaVersion": "1.0.0",
+  "invalid-field": "123",
+  "plugins": {
+    "plugin1": {
+      "random-field": 1
+    },
+    "plugin2": {
+      "version": "v1.0"
+    }
+  }
+}

pkg/porter/testdata/plugins.json

@@ -1,8 +1,12 @@
 {
-  "plugin1": {
-    "version": "v1.0"
-  },
-  "plugin2": {
-    "version": "v1.0"
+  "schemaType": "Plugins",
+  "schemaVersion": "1.0.0",
+  "plugins": {
+    "plugin1": {
+      "version": "v1.0"
+    },
+    "plugin2": {
+      "version": "v1.0"
+    }
   }
 }

pkg/porter/testdata/plugins.yaml

@@ -1,4 +1,7 @@
-plugin1:
-  version: v1.0
-plugin2:
-  version: v1.0
+schemaType: Plugins
+schemaVersion: 1.0.0
+plugins:
+  plugin1:
+    version: v1.0
+  plugin2:
+    version: v1.0