feat(python): allow default version to be overridden (#4206)

Adds PythonPackage.DefaultVersion which is populated during migration,
if and only if it is not the same as the final part of the first API
path, and then uses this override in repo metadata generation.

Additionally, the toAPIs code in the migration tool now sorts the APIs
by path before returning the list. This means that migration can process
the APIs in the same order they'll be saved at the end.

Fixes #3146

Author: jskeet

doc/config-schema.md

@@ -157,6 +157,7 @@ This document describes the schema for the librarian.yaml.
 | `name_pretty_override` | string | Allows the "name_pretty" field in .repo-metadata.json to be overridden, to reduce diffs while migrating. TODO(https://github.com/googleapis/librarian/issues/4175): remove this field. |
 | `product_documentation_override` | string | Allows the "product_documentation" field in .repo-metadata.json to be overridden, to reduce diffs while migrating. TODO(https://github.com/googleapis/librarian/issues/4175): remove this field. |
 | `metadata_name_override` | string | Allows the name in .repo-metadata.json (which is also used as part of the client documentation URI) to be overridden. By default it's the package name, but older packages use the API short name instead. |
+| `default_version` | string | Is the default version of the API to use. When omitted, the version in the first API path is used. |
 
 ## RustCrate Configuration
 

internal/config/language.go

@@ -324,6 +324,10 @@ type PythonPackage struct {
 	// default it's the package name, but older packages use the API short name
 	// instead.
 	MetadataNameOverride string `yaml:"metadata_name_override,omitempty"`
+
+	// DefaultVersion is the default version of the API to use. When omitted,
+	// the version in the first API path is used.
+	DefaultVersion string `yaml:"default_version,omitempty"`
 }
 
 // PythonDefault contains Python-specific default configuration.

internal/librarian/python/generate.go

@@ -108,24 +108,27 @@ func generate(ctx context.Context, config *config.Config, library *config.Librar
 
 // createRepoMetadata creates (in memory, not on disk) a RepoMetadata suitable
 // for the given library.
-func createRepoMetadata(config *config.Config, library *config.Library, googleapisDir string) (*repometadata.RepoMetadata, error) {
-	// TODO(https://github.com/googleapis/librarian/issues/3157):
-	// Copy files from .librarian/generator-input/client-post-processing
-	// for post processing, or reimplement.
-	repoMetadata, err := repometadata.FromLibrary(config, library, googleapisDir)
+func createRepoMetadata(cfg *config.Config, library *config.Library, googleapisDir string) (*repometadata.RepoMetadata, error) {
+	// Just to avoid lots of checks for library.Python being nil.
+	packageOptions := library.Python
+	if packageOptions == nil {
+		packageOptions = &config.PythonPackage{}
+	}
+	repoMetadata, err := repometadata.FromLibrary(cfg, library, googleapisDir)
 	if err != nil {
 		return nil, err
 	}
-	if library.Python != nil && library.Python.MetadataNameOverride != "" {
-		repoMetadata.Name = library.Python.MetadataNameOverride
+	if packageOptions.MetadataNameOverride != "" {
+		repoMetadata.Name = packageOptions.MetadataNameOverride
 	} else {
 		repoMetadata.Name = library.Name
 	}
-	// TODO(https://github.com/googleapis/librarian/issues/3146):
-	// Remove the default version fudge here, as Generate should
-	// compute it. For now, use the last component of the first api path as
-	// the default version.
+	// Use the version of the first-listed API path as the default version,
+	// unless it's overridden.
 	repoMetadata.DefaultVersion = filepath.Base(library.APIs[0].Path)
+	if packageOptions.DefaultVersion != "" {
+		repoMetadata.DefaultVersion = packageOptions.DefaultVersion
+	}
 	// TODO(https://github.com/googleapis/librarian/issues/4147): use the right
 	// library type.
 	repoMetadata.LibraryType = repometadata.GAPICAutoLibraryType
@@ -137,11 +140,11 @@ func createRepoMetadata(config *config.Config, library *config.Library, googleap
 	}
 	repoMetadata.ClientDocumentation = fmt.Sprintf(docTemplate, repoMetadata.Name)
 	// TODO(https://github.com/googleapis/librarian/issues/4175): remove these.
-	if library.Python != nil && library.Python.NamePrettyOverride != "" {
-		repoMetadata.NamePretty = library.Python.NamePrettyOverride
+	if packageOptions.NamePrettyOverride != "" {
+		repoMetadata.NamePretty = packageOptions.NamePrettyOverride
 	}
-	if library.Python != nil && library.Python.ProductDocumentationOverride != "" {
-		repoMetadata.ProductDocumentation = library.Python.ProductDocumentationOverride
+	if packageOptions.ProductDocumentationOverride != "" {
+		repoMetadata.ProductDocumentation = packageOptions.ProductDocumentationOverride
 	}
 	return repoMetadata, nil
 }

internal/librarian/python/generate_test.go

@@ -765,9 +765,8 @@ func TestCreateRepoMetadata(t *testing.T) {
 				Name:         "google-cloud-secret-manager",
 				ReleaseLevel: "stable",
 				APIs: []*config.API{
-					{
-						Path: "google/cloud/secretmanager/v1",
-					},
+					{Path: "google/cloud/secretmanager/v1"},
+					{Path: "google/cloud/secrets/v1beta1"},
 				},
 			},
 			want: &repometadata.RepoMetadata{
@@ -822,11 +821,11 @@ func TestCreateRepoMetadata(t *testing.T) {
 				ReleaseLevel:        "stable",
 				DescriptionOverride: "overridden description",
 				APIs: []*config.API{
-					{
-						Path: "google/cloud/secretmanager/v1",
-					},
+					{Path: "google/cloud/secretmanager/v1"},
+					{Path: "google/cloud/secrets/v1beta1"},
 				},
 				Python: &config.PythonPackage{
+					DefaultVersion:               "v1beta1",
 					MetadataNameOverride:         "secretmanager",
 					NamePrettyOverride:           "overridden name_pretty",
 					ProductDocumentationOverride: "overridden product_documentation",
@@ -846,6 +845,34 @@ func TestCreateRepoMetadata(t *testing.T) {
 				APIDescription:       "overridden description",
 				LibraryType:          "GAPIC_AUTO",
 				ClientDocumentation:  "https://cloud.google.com/python/docs/reference/secretmanager/latest",
+				DefaultVersion:       "v1beta1",
+			},
+		},
+		{
+			name: "default version",
+			library: &config.Library{
+				Name:         "google-cloud-secret-manager",
+				ReleaseLevel: "stable",
+				APIs: []*config.API{
+					{
+						Path: "google/cloud/secretmanager/v1",
+					},
+				},
+			},
+			want: &repometadata.RepoMetadata{
+				Name:                 "google-cloud-secret-manager",
+				NamePretty:           "Secret Manager",
+				ProductDocumentation: "https://cloud.google.com/secret-manager/",
+				IssueTracker:         "https://issuetracker.google.com/issues/new?component=784854&template=1380926",
+				ReleaseLevel:         "stable",
+				Language:             "python",
+				Repo:                 "googleapis/google-cloud-python",
+				DistributionName:     "google-cloud-secret-manager",
+				APIID:                "secretmanager.googleapis.com",
+				APIShortname:         "secretmanager",
+				APIDescription:       "Stores sensitive data such as API keys, passwords, and certificates.\nProvides convenience while improving security.",
+				LibraryType:          "GAPIC_AUTO",
+				ClientDocumentation:  "https://cloud.google.com/python/docs/reference/google-cloud-secret-manager/latest",
 				DefaultVersion:       "v1",
 			},
 		},

tool/cmd/migrate/legacylibrarian.go

@@ -24,6 +24,7 @@ import (
 	"reflect"
 	"slices"
 	"sort"
+	"strings"
 
 	"github.com/bazelbuild/buildtools/build"
 	"github.com/googleapis/librarian/internal/config"
@@ -343,6 +344,12 @@ func toAPIs(legacyapis []*legacyconfig.API) []*config.API {
 			Path: api.Path,
 		})
 	}
+	// Formatting the library will sort the APIs by path later anyway, so let's
+	// do that now. That way the migration code will observe the list of APIs
+	// in the same order that it will eventually be saved.
+	slices.SortFunc(apis, func(a, b *config.API) int {
+		return strings.Compare(a.Path, b.Path)
+	})
 	return apis
 }
 

tool/cmd/migrate/legacylibrarian_test.go

@@ -720,3 +720,18 @@ func TestParseBazel_Error(t *testing.T) {
 		})
 	}
 }
+
+func TestToAPIs(t *testing.T) {
+	legacyAPIs := []*legacyconfig.API{
+		{Path: "google/cloud/functions/v2"},
+		{Path: "google/cloud/functions/v1"},
+	}
+	want := []*config.API{
+		{Path: "google/cloud/functions/v1"},
+		{Path: "google/cloud/functions/v2"},
+	}
+	got := toAPIs(legacyAPIs)
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("mismatch (-want +got):\n%s", diff)
+	}
+}

tool/cmd/migrate/python.go

@@ -299,6 +299,7 @@ func filterPathsByRegex(paths []string, regexps []*regexp.Regexp) []string {
 func applyRepoMetadata(metadataPath, googleapisDir string, library *config.Library) (*config.Library, error) {
 	defaultTitle := ""
 	defaultDocumentationURI := ""
+	defaultDefaultVersion := ""
 	// Load the service config file for the first API if there is one, and
 	// use that
 	if len(library.APIs) > 0 {
@@ -308,6 +309,7 @@ func applyRepoMetadata(metadataPath, googleapisDir string, library *config.Libra
 		}
 		defaultTitle = apiInfo.Title
 		defaultDocumentationURI = apiInfo.DocumentationURI
+		defaultDefaultVersion = filepath.Base(library.APIs[0].Path)
 	}
 
 	// Load the current repo metadata and apply overrides for anything that
@@ -338,6 +340,9 @@ func applyRepoMetadata(metadataPath, googleapisDir string, library *config.Libra
 	if repoMetadata.Name != library.Name {
 		library.Python.MetadataNameOverride = repoMetadata.Name
 	}
+	if repoMetadata.DefaultVersion != defaultDefaultVersion {
+		library.Python.DefaultVersion = repoMetadata.DefaultVersion
+	}
 
 	return library, nil
 }

tool/cmd/migrate/python_test.go

@@ -128,7 +128,7 @@ func TestBuildPythonLibraries(t *testing.T) {
 			},
 		},
 		{
-			name: "billing budgets (transport varies by API)",
+			name: "billing budgets (transport varies by API, need explicit default)",
 			input: &MigrationInput{
 				repoPath: "testdata/google-cloud-python",
 				librarianState: &legacyconfig.LibrarianState{
@@ -153,6 +153,7 @@ func TestBuildPythonLibraries(t *testing.T) {
 					},
 					DescriptionOverride: "The Cloud Billing Budget API stores Cloud Billing budgets, which define a budget plan and the rules to execute as spend is tracked against that plan.",
 					Python: &config.PythonPackage{
+						DefaultVersion:               "v1beta",
 						NamePrettyOverride:           "Cloud Billing Budget",
 						ProductDocumentationOverride: "https://cloud.google.com/billing/docs/how-to/budget-api-overview",
 						OptArgsByAPI: map[string][]string{

tool/cmd/migrate/testdata/google-cloud-python/packages/google-cloud-audit-log/.repo-metadata.json

@@ -2,7 +2,7 @@
   "api_id": "",
   "api_shortname": "auditlog",
   "client_documentation": "https://github.com/googleapis/google-cloud-python/tree/main/packages/google-cloud-audit-log",
-  "default_version": "apiVersion",
+  "default_version": "",
   "distribution_name": "google-cloud-audit-log",
   "issue_tracker": "",
   "language": "python",

tool/cmd/migrate/testdata/google-cloud-python/packages/google-cloud-billing-budgets/.repo-metadata.json

@@ -3,7 +3,7 @@
   "api_id": "billingbudgets.googleapis.com",
   "api_shortname": "billingbudgets",
   "client_documentation": "https://cloud.google.com/python/docs/reference/google-cloud-billing-budgets/latest",
-  "default_version": "v1",
+  "default_version": "v1beta",
   "distribution_name": "google-cloud-billing-budgets",
   "issue_tracker": "https://issuetracker.google.com/savedsearches/559770",
   "language": "python",