Sync feature branch resource identity

.ci/changelog2.tmpl

@@ -0,0 +1,50 @@
+{{- if .NotesByType.unknown -}}
+UNKNOWN CHANGELOG TYPE:
+{{range .NotesByType.unknown -}}
+* {{ template "note" .}}
+{{ end -}}
+{{- end -}}
+
+{{- if .NotesByType.note -}}
+NOTES:
+{{range .NotesByType.note -}}
+* {{ template "note" .}}
+{{ end -}}
+{{- end -}}
+
+{{- if .NotesByType.deprecation -}}
+DEPRECATIONS:
+{{range .NotesByType.deprecation -}}
+* {{ template "note" .}}
+{{ end -}}
+{{- end -}}
+
+{{- if index .NotesByType "breaking-change" -}}
+BREAKING CHANGES:
+{{range index .NotesByType "breaking-change" -}}
+* {{ template "note" .}}
+{{ end -}}
+{{- end -}}
+
+{{- $features := combineTypes .NotesByType.feature (index .NotesByType "new-resource" ) (index .NotesByType "new-datasource") (index .NotesByType "new-data-source") (index .NotesByType "new-function" ) (index .NotesByType "new-ephemeral" ) -}}
+{{- if $features }}
+FEATURES:
+{{range $features | sort -}}
+* {{ template "note" . }}
+{{ end -}}
+{{- end -}}
+
+{{- $improvements := combineTypes .NotesByType.improvement .NotesByType.enhancement -}}
+{{- if $improvements }}
+IMPROVEMENTS:
+{{range $improvements | sort -}}
+* {{ template "note" . }}
+{{ end -}}
+{{- end -}}
+
+{{- if .NotesByType.bug }}
+BUG FIXES:
+{{range .NotesByType.bug -}}
+* {{ template "note" . }}
+{{ end -}}
+{{- end -}}

.ci/containers/build-environment/Dockerfile

@@ -25,7 +25,7 @@ ENV GO111MODULE "on"
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 
-RUN apt-get update && apt-get install -y --no-install-recommends git openssh-client apt-transport-https ca-certificates curl netbase wget gcc make jq libjq1
+RUN apt-get update && apt-get install -y --no-install-recommends git openssh-client apt-transport-https ca-certificates curl netbase wget gcc make jq libjq1 unzip zip
 
 RUN git config --global user.name "Modular Magician"
 RUN git config --global user.email "magic-modules@google.com"

.ci/containers/go-plus/Dockerfile

@@ -21,7 +21,7 @@ COPY --from=builder /go/pkg/mod /go/pkg/mod
 COPY --from=builder /go/cache /go/cache
 
 RUN apt-get update && \
-    apt-get install -y git jq unzip parallel curl && \
+    apt-get install -y git jq unzip zip parallel curl && \
     echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
     curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg  add - && \
     apt-get update -y && \
@@ -32,4 +32,4 @@ RUN apt-get update && \
 RUN wget https://releases.hashicorp.com/terraform/1.11.0/terraform_1.11.0_linux_amd64.zip \
     && unzip terraform_1.11.0_linux_amd64.zip \
     && rm terraform_1.11.0_linux_amd64.zip \
-    && mv ./terraform /bin/terraform
+    && mv ./terraform /bin/terraform

.ci/infra/terraform/main.tf

@@ -306,6 +306,7 @@ module "project-services" {
     "migrationcenter.googleapis.com",
     "ml.googleapis.com",
     "mobilecrashreporting.googleapis.com",
+    "modelarmor.googleapis.com",
     "monitoring.googleapis.com",
     "multiclustermetering.googleapis.com",
     "netapp.googleapis.com",

.ci/magician/cmd/generate_comment.go

@@ -16,6 +16,7 @@
 package cmd
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -35,6 +36,9 @@ import (
 	"github.com/GoogleCloudPlatform/magic-modules/tools/issue-labeler/labeler"
 
 	"github.com/spf13/cobra"
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/parser"
+	"go.abhg.dev/goldmark/frontmatter"
 	"golang.org/x/exp/maps"
 
 	_ "embed"
@@ -345,6 +349,11 @@ func execGenerateComment(prNumber int, ghTokenMagicModules, buildId, buildStep,
 				errors[repo.Title] = append(errors[repo.Title], "The missing doc detector failed to run.")
 			}
 			data.MissingDocs = missingDocs
+
+			errStrs := checkDocumentFrontmatter(repo)
+			if len(errStrs) > 0 {
+				errors[repo.Title] = append(errors[repo.Title], errStrs...)
+			}
 		}
 
 		simpleDiff, err := computeAffectedResources(diffProcessorPath, rnr, repo)
@@ -670,3 +679,49 @@ func pathChanged(path string, changedFiles []string) bool {
 func init() {
 	rootCmd.AddCommand(generateCommentCmd)
 }
+
+// checkDocumentFrontmatter checks changed markdown files' frontmatter
+// structure in the repo and returns error strings when applicable.
+func checkDocumentFrontmatter(repo source.Repo) []string {
+	var errs []string
+	for _, f := range repo.ChangedFiles {
+		if !strings.HasSuffix(f, ".markdown") {
+			continue
+		}
+		src, err := os.ReadFile(filepath.Join(repo.Path, f))
+		if err != nil {
+			errs = append(errs, "Error reading file "+f)
+			continue
+		}
+
+		md := goldmark.New(
+			goldmark.WithExtensions(&frontmatter.Extender{}),
+		)
+
+		ctx := parser.NewContext()
+		var buff bytes.Buffer
+
+		err = md.Convert(src, &buff, parser.WithContext(ctx))
+		if err != nil {
+			errs = append(errs, "Error parsing file "+f)
+			continue
+		}
+		data := frontmatter.Get(ctx)
+		if data == nil {
+			errs = append(errs, fmt.Sprintf("No frontmatter found in file %s. This is usually due to an incorrect structure in the frontmatter.", f))
+			continue
+		}
+
+		var metadata struct {
+			Subcategory string
+		}
+		if err := data.Decode(&metadata); err != nil {
+			errs = append(errs, fmt.Sprintf("Failed to decode frontmatter in file %s. This is usually due to an incorrect structure in the frontmatter.", f))
+			continue
+		}
+		if metadata.Subcategory == "" {
+			errs = append(errs, fmt.Sprintf("Failed to detect subcategory in the frontmatter in file %s.", f))
+		}
+	}
+	return errs
+}

.ci/magician/cmd/generate_comment_test.go

@@ -18,6 +18,7 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"reflect"
 	"testing"
 
@@ -550,3 +551,78 @@ func TestPathChanged(t *testing.T) {
 		})
 	}
 }
+
+func TestCheckDocumentFrontmatter(t *testing.T) {
+	tmpDir := t.TempDir()
+	files := map[string]string{
+		"malformed.markdown": `
+subcategory: Example Subcategory
+---	
+`,
+		"sample.markdown": `
+---
+subcategory: Example Subcategory
+---	
+`,
+		"missingsubcategory.markdown": `
+---
+random: Example Subcategory
+---	
+`,
+	}
+	for name, content := range files {
+		fullPath := filepath.Join(tmpDir, name)
+		err := os.WriteFile(fullPath, []byte(content), 0644)
+		if err != nil {
+			t.Fatalf("Failed to create file %s: %v", name, err)
+		}
+	}
+
+	tests := []struct {
+		name         string
+		changedFiles []string
+		wantErr      bool
+	}{
+		{
+			name:         "No changed markdown files",
+			changedFiles: []string{"abc.txt"},
+			wantErr:      false,
+		},
+		{
+			name:         "malformed markdown",
+			changedFiles: []string{"malformed.markdown"},
+			wantErr:      true,
+		},
+		{
+			name:         "not exist markdown",
+			changedFiles: []string{"abc.markdown"},
+			wantErr:      true,
+		},
+		{
+			name:         "Changed files with no frontmatter",
+			changedFiles: []string{"sample.markdown"},
+			wantErr:      false,
+		},
+		{
+			name:         "Missing subcategory in frontmatter",
+			changedFiles: []string{"missingsubcategory.markdown"},
+			wantErr:      true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			repo := source.Repo{
+				Path:         tmpDir,
+				ChangedFiles: tc.changedFiles,
+			}
+			got := checkDocumentFrontmatter(repo)
+			if tc.wantErr && len(got) == 0 {
+				t.Errorf("checkDocumentFrontmatter() = %v, want error", got)
+			}
+			if !tc.wantErr && len(got) > 0 {
+				t.Errorf("checkDocumentFrontmatter() = %v, want no error", got)
+			}
+		})
+	}
+}

.ci/magician/cmd/generate_downstream.go

@@ -383,6 +383,12 @@ func addChangelogEntry(downstreamRepo *source.Repo, pullRequest *github.PullRequ
 			return err
 		}
 	}
+	// If changelog entry is missing, add an entry "unknown: <PR title>".
+	if matches == nil {
+		if err := rnr.WriteFile(filepath.Join(".changelog", fmt.Sprintf("%d.txt", pullRequest.Number)), "unknown: "+pullRequest.Title); err != nil {
+			return err
+		}
+	}
 	return rnr.PopDir()
 }
 

.ci/magician/cmd/templates/vcr/post_replay.tmpl

@@ -0,0 +1,55 @@
+{{- if or (gt (len .NotRunBetaTests) 0) (gt (len .NotRunGATests) 0)}}
+#### Non-exercised tests
+
+{{if gt (len .NotRunBetaTests) 0 -}}
+{{color "red" "Tests were added that are skipped in VCR:"}}
+{{range .NotRunBetaTests}}{{. | printf "- %s\n"}}{{end}}
+{{end}}
+
+{{if gt (len .NotRunGATests) 0 -}}
+{{color "red" "Tests were added that are GA-only additions and require manual runs:"}}
+{{range .NotRunGATests}}{{. | printf "- %s\n"}}{{end}}
+{{end}}
+{{end}}
+#### Tests analytics
+Total tests: {{add (add (len .ReplayingResult.PassedTests) (len .ReplayingResult.SkippedTests)) (len .ReplayingResult.FailedTests) }}
+Passed tests: {{len .ReplayingResult.PassedTests}}
+Skipped tests: {{len .ReplayingResult.SkippedTests}}
+Affected tests: {{len .ReplayingResult.FailedTests}}
+
+<details>
+<summary>Click here to see the affected service packages</summary>
+<blockquote>
+{{if .RunFullVCR}}
+All service packages are affected
+{{else if gt (len .AffectedServices) 0}}
+<ul>
+{{range .AffectedServices}}{{. | printf "<li>%s</li>\n"}}{{end}}
+</ul>
+{{else}}
+None
+{{end}}
+</blockquote>
+</details>
+{{ if gt (len .ReplayingResult.FailedTests) 0 -}}
+#### Action taken
+<details>
+<summary>Found {{len .ReplayingResult.FailedTests}} affected test(s) by replaying old test recordings. Starting RECORDING based on the most recent commit. Click here to see the affected tests
+</summary>
+<blockquote>
+<ul>
+{{range .ReplayingResult.FailedTests}}{{. | printf "<li>%s</li>\n"}}{{end}}
+</ul>
+</blockquote>
+</details>
+
+[Get to know how VCR tests work](https://googlecloudplatform.github.io/magic-modules/develop/test/test/)
+{{ else -}}
+{{- if .ReplayingErr -}}
+{{color "red" "Errors occurred during REPLAYING mode. Please fix them to complete your PR."}}
+{{- else -}}
+{{color "green" "All tests passed!"}}
+{{- end}}
+
+View the [build log](https://storage.cloud.google.com/{{.LogBucket}}/{{.Version}}/refs/heads/{{.Head}}/artifacts/{{.BuildID}}/build-log/replaying_test.log)
+{{- end}}

.ci/magician/cmd/templates/vcr/record_replay.tmpl

@@ -2,7 +2,7 @@
 {{color "green" "Tests passed during RECORDING mode:"}}
 {{range .RecordingResult.PassedTests -}}
 `{{.}}` {{/* remove trailing whitespace */ -}}
-  [[Debug log](https://storage.cloud.google.com/{{$.LogBucket}}/{{$.Version}}/refs/heads/{{$.Head}}/artifacts/{{$.BuildID}}/recording/{{.}}.log)]
+  [[Debug log]({{$.LogBaseUrl}}/recording/{{.}}.log)]
 {{/* remove trailing whitespace */ -}}
 {{end}}
 
@@ -11,8 +11,8 @@
 {{color "red" "Tests failed when rerunning REPLAYING mode:"}}
 {{range .ReplayingAfterRecordingResult.FailedTests -}}
 `{{.}}` {{/* remove trailing whitespace */ -}}
-  [[Error message](https://storage.cloud.google.com/{{$.LogBucket}}/{{$.Version}}/refs/heads/{{$.Head}}/artifacts/{{$.BuildID}}/build-log/replaying_build_after_recording/{{.}}_replaying_test.log)] {{/* remove trailing whitespace */ -}}
-  [[Debug log](https://storage.cloud.google.com/{{$.LogBucket}}/{{$.Version}}/refs/heads/{{$.Head}}/artifacts/{{$.BuildID}}/replaying_after_recording/{{.}}.log)]
+  [[Error message]({{$.LogBaseUrl}}/build-log/replaying_build_after_recording/{{compoundTest .}}_replaying_test.log)] {{/* remove trailing whitespace */ -}}
+  [[Debug log]({{$.LogBaseUrl}}/replaying_after_recording/{{.}}.log)]
 {{/* remove trailing whitespace */ -}}
 {{end}}
 
@@ -30,8 +30,8 @@ Please fix these to complete your PR. If you believe these test failures to be i
 {{color "red" "Tests failed during RECORDING mode:"}}
 {{range .RecordingResult.FailedTests -}}
 `{{.}}` {{/* remove trailing whitespace */ -}}
-  [[Error message](https://storage.cloud.google.com/{{$.LogBucket}}/{{$.Version}}/refs/heads/{{$.Head}}/artifacts/{{$.BuildID}}/build-log/recording_build/{{.}}_recording_test.log)] {{/* remove trailing whitespace */ -}}
-  [[Debug log](https://storage.cloud.google.com/{{$.LogBucket}}/{{$.Version}}/refs/heads/{{$.Head}}/artifacts/{{$.BuildID}}/recording/{{.}}.log)]
+  [[Error message]({{$.LogBaseUrl}}/build-log/recording_build/{{compoundTest .}}_recording_test.log)] {{/* remove trailing whitespace */ -}}
+  [[Debug log]({{$.LogBaseUrl}}/recording/{{.}}.log)]
 {{/* remove trailing whitespace */ -}}
 {{end}}
 {{end}} {{- /* end of if gt (len .RecordingResult.FailedTests) 0 */ -}}
@@ -42,5 +42,5 @@ Please fix these to complete your PR. If you believe these test failures to be i
 
 {{if .AllRecordingPassed}}{{color "green" "All tests passed!"}}{{end}}
 
-View the [build log](https://storage.cloud.google.com/{{.LogBucket}}/{{.Version}}/refs/heads/{{.Head}}/artifacts/{{.BuildID}}/build-log/recording_test.log) {{/* remove trailing whitespace */ -}}
-or the [debug log](https://console.cloud.google.com/storage/browser/{{.LogBucket}}/{{.Version}}/refs/heads/{{.Head}}/artifacts/{{.BuildID}}/recording) for each test
+View the [build log]({{.LogBaseUrl}}/build-log/recording_test.log) {{/* remove trailing whitespace */ -}}
+or the [debug log]({{.BrowseLogBaseUrl}}/recording) for each test