feat(python): support proto-only APIs (#4075)

Adds handling for "proto-only" APIs (by which we mean APIs without a
GAPIC library configured). These APIs:

- Use the python and pyi protoc plugins, not gapic
- Generate to a slightly different staging directory
- Copy the .proto files as well as the generated code

This change configures these APIs during migration.

Fixes #3204.

Author: jskeet

doc/config-schema.md

@@ -103,7 +103,7 @@ This document describes the schema for the librarian.yaml.
 
 ## DartPackage Configuration
 
-[Link to code](../internal/config/language.go#L284)
+[Link to code](../internal/config/language.go#L288)
 | Field | Type | Description |
 | :--- | :--- | :--- |
 | `api_keys_environment_variables` | string | APIKeysEnvironmentVariables is a comma-separated list of environment variable names that can contain API keys (e.g., "GOOGLE_API_KEY,GEMINI_API_KEY"). |
@@ -152,6 +152,7 @@ This document describes the schema for the librarian.yaml.
 | :--- | :--- | :--- |
 | `opt_args` | list of string | OptArgs contains additional options passed to the generator, where the options are common to all apis. Example: ["warehouse-package-name=google-cloud-batch"] |
 | `opt_args_by_api` | map[string][]string | OptArgsByAPI contains additional options passed to the generator, where the options vary by api. In each entry, the key is the api (API path) and the value is the list of options to pass when generating that API. Example: {"google/cloud/secrets/v1beta": ["python-gapic-name=secretmanager"]} |
+| `proto_only_apis` | list of string | ProtoOnlyAPIs contains the list of API paths which are proto-only, so should use regular protoc Python generation instead of GAPIC. |
 
 ## RustCrate Configuration
 

internal/config/language.go

@@ -278,6 +278,10 @@ type PythonPackage struct {
 	// that API.
 	// Example: {"google/cloud/secrets/v1beta": ["python-gapic-name=secretmanager"]}
 	OptArgsByAPI map[string][]string `yaml:"opt_args_by_api,omitempty"`
+
+	// ProtoOnlyAPIs contains the list of API paths which are proto-only, so
+	// should use regular protoc Python generation instead of GAPIC.
+	ProtoOnlyAPIs []string `yaml:"proto_only_apis,omitempty"`
 }
 
 // DartPackage contains Dart-specific library configuration.

internal/librarian/python/generate.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"slices"
 	"strings"
 
 	"github.com/googleapis/librarian/internal/config"
@@ -101,7 +102,8 @@ func generateAPI(ctx context.Context, api *config.API, library *config.Library,
 	// TODO(https://github.com/googleapis/librarian/issues/3210): generate
 	// directly in place.
 
-	stagingChildDirectory := getStagingChildDirectory(api.Path)
+	protoOnly := isProtoOnly(api, library)
+	stagingChildDirectory := getStagingChildDirectory(api.Path, protoOnly)
 	stagingDir := filepath.Join(repoRoot, "owl-bot-staging", library.Name, stagingChildDirectory)
 	if err := os.MkdirAll(stagingDir, 0755); err != nil {
 		return err
@@ -141,10 +143,42 @@ func generateAPI(ctx context.Context, api *config.API, library *config.Library,
 		return fmt.Errorf("%s: %w", cmd.String(), err)
 	}
 
+	// Copy the proto files as well as the generated code for proto-only libraries.
+	if protoOnly {
+		if err := stageProtoFiles(googleapisDir, stagingDir, protos); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func stageProtoFiles(googleapisDir, targetDir string, relativeProtoPaths []string) error {
+	for _, proto := range relativeProtoPaths {
+		sourceProtoFile := filepath.Join(googleapisDir, proto)
+		content, err := os.ReadFile(sourceProtoFile)
+		if err != nil {
+			return fmt.Errorf("reading proto file %s failed: %w", sourceProtoFile, err)
+		}
+		targetProtoFile := filepath.Join(targetDir, proto)
+		dir := filepath.Dir(targetProtoFile)
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			return fmt.Errorf("creating directory %s failed: %w", dir, err)
+		}
+		if err := os.WriteFile(targetProtoFile, content, 0644); err != nil {
+			return fmt.Errorf("writing proto file %s failed: %w", targetProtoFile, err)
+		}
+	}
 	return nil
 }
 
-func createProtocOptions(ch *config.API, library *config.Library, googleapisDir, stagingDir string) ([]string, error) {
+func createProtocOptions(api *config.API, library *config.Library, googleapisDir, stagingDir string) ([]string, error) {
+	if isProtoOnly(api, library) {
+		return []string{
+			fmt.Sprintf("--python_out=%s", stagingDir),
+			fmt.Sprintf("--pyi_out=%s", stagingDir),
+		}, nil
+	}
 	// GAPIC library: generate full client library
 	opts := []string{"metadata"}
 
@@ -155,7 +189,7 @@ func createProtocOptions(ch *config.API, library *config.Library, googleapisDir,
 	}
 	// Then options that apply to this specific api
 	if library.Python != nil && len(library.Python.OptArgsByAPI) > 0 {
-		apiOptArgs, ok := library.Python.OptArgsByAPI[ch.Path]
+		apiOptArgs, ok := library.Python.OptArgsByAPI[api.Path]
 		if ok {
 			opts = append(opts, apiOptArgs...)
 		}
@@ -187,7 +221,7 @@ func createProtocOptions(ch *config.API, library *config.Library, googleapisDir,
 	}
 
 	// Add gRPC service config (retry/timeout settings)
-	grpcConfigPath, err := serviceconfig.FindGRPCServiceConfig(googleapisDir, ch.Path)
+	grpcConfigPath, err := serviceconfig.FindGRPCServiceConfig(googleapisDir, api.Path)
 	if err != nil {
 		return nil, err
 	}
@@ -200,12 +234,12 @@ func createProtocOptions(ch *config.API, library *config.Library, googleapisDir,
 		opts = append(opts, fmt.Sprintf("retry-config=%s", grpcConfigPath))
 	}
 
-	api, err := serviceconfig.Find(googleapisDir, ch.Path, serviceconfig.LangPython)
+	apiMetadata, err := serviceconfig.Find(googleapisDir, api.Path, serviceconfig.LangPython)
 	if err != nil {
 		return nil, err
 	}
-	if api != nil && api.ServiceConfig != "" {
-		opts = append(opts, fmt.Sprintf("service-yaml=%s", api.ServiceConfig))
+	if apiMetadata != nil && apiMetadata.ServiceConfig != "" {
+		opts = append(opts, fmt.Sprintf("service-yaml=%s", apiMetadata.ServiceConfig))
 	}
 
 	return []string{
@@ -214,13 +248,17 @@ func createProtocOptions(ch *config.API, library *config.Library, googleapisDir,
 	}, nil
 }
 
+func isProtoOnly(api *config.API, library *config.Library) bool {
+	return library.Python != nil && slices.Contains(library.Python.ProtoOnlyAPIs, api.Path)
+}
+
 // getStagingChildDirectory determines where within owl-bot-staging/{library-name} the
 // generated code the given API path should be staged. This is not quite equivalent
 // to _get_staging_child_directory in the Python container, as for proto-only directories
 // we don't want the apiPath suffix.
-func getStagingChildDirectory(apiPath string) string {
+func getStagingChildDirectory(apiPath string, isProtoOnly bool) string {
 	versionCandidate := filepath.Base(apiPath)
-	if strings.HasPrefix(versionCandidate, "v") {
+	if strings.HasPrefix(versionCandidate, "v") || isProtoOnly {
 		return versionCandidate
 	} else {
 		return versionCandidate + "-py"

internal/librarian/python/generate_test.go

@@ -15,10 +15,13 @@
 package python
 
 import (
+	"errors"
 	"fmt"
+	"io/fs"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"syscall"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -31,9 +34,10 @@ const googleapisDir = "../../testdata/googleapis"
 func TestGetStagingChildDirectory(t *testing.T) {
 	t.Parallel()
 	for _, test := range []struct {
-		name     string
-		apiPath  string
-		expected string
+		name      string
+		apiPath   string
+		protoOnly bool
+		expected  string
 	}{
 		{
 			name:     "versioned path",
@@ -45,9 +49,15 @@ func TestGetStagingChildDirectory(t *testing.T) {
 			apiPath:  "google/cloud/secretmanager/type",
 			expected: "type-py",
 		},
+		{
+			name:      "proto-only",
+			apiPath:   "google/cloud/secretmanager/type",
+			protoOnly: true,
+			expected:  "type",
+		},
 	} {
 		t.Run(test.name, func(t *testing.T) {
-			got := getStagingChildDirectory(test.apiPath)
+			got := getStagingChildDirectory(test.apiPath, test.protoOnly)
 			if diff := cmp.Diff(test.expected, got); diff != "" {
 				t.Errorf("getStagingChildDirectory(%q) returned diff (-want +got):\n%s", test.apiPath, diff)
 			}
@@ -188,6 +198,32 @@ func TestCreateProtocOptions(t *testing.T) {
 				"--python_gapic_opt=metadata,transport=rest,rest-numeric-enums,retry-config=google/cloud/secretmanager/v1/secretmanager_grpc_service_config.json,service-yaml=google/cloud/secretmanager/v1/secretmanager_v1.yaml",
 			},
 		},
+		{
+			name: "proto-only exists but doesn't include API path",
+			api:  &config.API{Path: "google/cloud/secretmanager/v1"},
+			library: &config.Library{
+				Python: &config.PythonPackage{
+					ProtoOnlyAPIs: []string{"google/cloud/secretmanager/type"},
+				},
+			},
+			expected: []string{
+				"--python_gapic_out=staging",
+				"--python_gapic_opt=metadata,rest-numeric-enums,retry-config=google/cloud/secretmanager/v1/secretmanager_grpc_service_config.json,service-yaml=google/cloud/secretmanager/v1/secretmanager_v1.yaml",
+			},
+		},
+		{
+			name: "proto-only exists and includes API path",
+			api:  &config.API{Path: "google/cloud/secretmanager/type"},
+			library: &config.Library{
+				Python: &config.PythonPackage{
+					ProtoOnlyAPIs: []string{"google/cloud/secretmanager/type"},
+				},
+			},
+			expected: []string{
+				"--python_out=staging",
+				"--pyi_out=staging",
+			},
+		},
 	} {
 		t.Run(test.name, func(t *testing.T) {
 			got, err := createProtocOptions(test.api, test.library, googleapisDir, "staging")
@@ -202,6 +238,86 @@ func TestCreateProtocOptions(t *testing.T) {
 	}
 }
 
+func TestStageProtoFiles(t *testing.T) {
+	targetDir := t.TempDir()
+	// Deliberately not including all proto files (or any non-proto) files here.
+	relativeProtoPaths := []string{
+		"google/cloud/gkehub/v1/feature.proto",
+		"google/cloud/gkehub/v1/membership.proto",
+	}
+	if err := stageProtoFiles(googleapisDir, targetDir, relativeProtoPaths); err != nil {
+		t.Fatal(err)
+	}
+	copiedFiles := []string{}
+	if err := filepath.WalkDir(targetDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if !d.Type().IsDir() {
+			relative, err := filepath.Rel(targetDir, path)
+			if err != nil {
+				return err
+			}
+			copiedFiles = append(copiedFiles, relative)
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(relativeProtoPaths, copiedFiles); diff != "" {
+		t.Errorf("mismatch (-want +got):\n%s", diff)
+	}
+}
+
+func TestStageProtoFiles_Error(t *testing.T) {
+	t.Parallel()
+	for _, test := range []struct {
+		name               string
+		relativeProtoPaths []string
+		setup              func(t *testing.T, targetDir string)
+		wantErr            error
+	}{
+		{
+			name:               "path doesn't exist",
+			relativeProtoPaths: []string{"google/cloud/bogus.proto"},
+			wantErr:            os.ErrNotExist,
+		},
+		{
+			name:               "can't create directory",
+			relativeProtoPaths: []string{"google/cloud/gkehub/v1/feature.proto"},
+			setup: func(t *testing.T, targetDir string) {
+				// Create a file with the name of the directory we'd create.
+				if err := os.WriteFile(filepath.Join(targetDir, "google"), []byte{}, 0644); err != nil {
+					t.Fatal(err)
+				}
+			},
+			wantErr: syscall.ENOTDIR,
+		},
+		{
+			name:               "can't write file",
+			relativeProtoPaths: []string{"google/cloud/gkehub/v1/feature.proto"},
+			setup: func(t *testing.T, targetDir string) {
+				// Create a directory with the name of the file we'd create.
+				if err := os.MkdirAll(filepath.Join(targetDir, "google", "cloud", "gkehub", "v1", "feature.proto"), 0755); err != nil {
+					t.Fatal(err)
+				}
+			},
+			wantErr: syscall.EISDIR,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			targetDir := t.TempDir()
+			if test.setup != nil {
+				test.setup(t, targetDir)
+			}
+			gotErr := stageProtoFiles(googleapisDir, targetDir, test.relativeProtoPaths)
+			if !errors.Is(gotErr, test.wantErr) {
+				t.Errorf("stageProtoFiles error = %v, wantErr %v", gotErr, test.wantErr)
+			}
+		})
+	}
+}
+
 func TestCopyReadmeToDocsDir(t *testing.T) {
 	t.Parallel()
 	for _, test := range []struct {

tool/cmd/migrate/python.go

@@ -18,7 +18,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/fs"
-	"log/slog"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -108,15 +107,14 @@ func applyBuildBazelConfig(library *config.Library, googleapisDir string) (*conf
 	}
 	allTransports := make(map[string]bool)
 	transportsByApi := make(map[string]string)
-	allGapic := true
 
 	for _, api := range library.APIs {
 		bazelGapicInfo, err := parseBazelPythonInfo(googleapisDir, api.Path)
 		if err != nil {
 			return nil, err
 		}
 		if bazelGapicInfo == nil {
-			allGapic = false
+			pythonConfig.ProtoOnlyAPIs = append(pythonConfig.ProtoOnlyAPIs, api.Path)
 			continue
 		}
 		transportsByApi[api.Path] = bazelGapicInfo.transport
@@ -125,26 +123,33 @@ func applyBuildBazelConfig(library *config.Library, googleapisDir string) (*conf
 			pythonConfig.OptArgsByAPI[api.Path] = bazelGapicInfo.optArgs
 		}
 	}
-	if !allGapic {
-		slog.Info("Skipping not-fully-GAPIC library", "library", library.Name)
-		return nil, nil
-	}
 	if len(allTransports) == 1 {
 		// One consistent transport; set it library-wide if it's not the default.
+		// This assumes that where there's a mixture of GAPIC and non-GAPIC, the
+		// first path is a GAPIC API, but that happens to be true for now (and
+		// we don't care what happens post-migration).
 		transport := transportsByApi[library.APIs[0].Path]
 		if transport != "grpc+rest" {
 			library.Transport = transport
 		}
 	} else {
-		// Transport differs by API version. Add it into OptArgsByAPI.
+		// Transport differs by API version. Add it into OptArgsByAPI, but only
+		// for non-proto-only APIs. (Proto-only APIs don't have a transport
+		// anyway.)
 		for _, api := range library.APIs {
+			if slices.Contains(pythonConfig.ProtoOnlyAPIs, api.Path) {
+				continue
+			}
 			optArgs := pythonConfig.OptArgsByAPI[api.Path]
 			optArgs = append(optArgs, fmt.Sprintf("transport=%s", transportsByApi[api.Path]))
 			pythonConfig.OptArgsByAPI[api.Path] = optArgs
 		}
 	}
 
-	if len(pythonConfig.OptArgsByAPI) > 0 {
+	if len(pythonConfig.OptArgsByAPI) > 0 || len(pythonConfig.ProtoOnlyAPIs) > 0 {
+		if len(pythonConfig.OptArgsByAPI) == 0 {
+			pythonConfig.OptArgsByAPI = nil
+		}
 		library.Python = pythonConfig
 	}
 	return library, nil

tool/cmd/migrate/python_test.go

@@ -107,6 +107,14 @@ func TestBuildPythonLibraries(t *testing.T) {
 				librarianConfig: &legacyconfig.LibrarianConfig{},
 			},
 			want: []*config.Library{
+				{
+					Name:         "google-cloud-audit-log",
+					APIs:         []*config.API{{Path: "google/cloud/audit"}},
+					ReleaseLevel: "preview",
+					Python: &config.PythonPackage{
+						ProtoOnlyAPIs: []string{"google/cloud/audit"},
+					},
+				},
 				{
 					Name:         "google-cloud-workstations",
 					ReleaseLevel: "preview",