feat(internal/surfer): add command groups (#3544)

## PR Summary: Generate `__init__.py` for proper command group detection

This PR updates the `gcloud` surface generator to include `__init__.py`
files in the generated directory structure. These files are required by
the `gcloud` CLI framework to correctly identify and load command groups
and their nested hierarchies.

### Key Changes

1.  **Generation Logic (`generate.go`):**
- Implemented a `writeInitPy` helper function to create empty package
markers.
- Updated `generateService` to ensure the service root directory (e.g.,
`parallelstore/`) contains an `__init__.py`.
- Updated `generateResourceCommands` to ensure each resource group
directory (e.g., `parallelstore/instances/`) contains an `__init__.py`.
- Fixed an issue where the resource directory was not created before
attempting to write the initialization file.

2.  **Verification:**
- Updated `TestGenerateResourceCommands` to verify the presence of
`__init__.py` in generated resource directories.
- Manually verified the generated file structure for the Parallelstore
API.

### File Structure Comparison

**Old Structure:**
```
parallelstore
└── instances
    ├── _partials/
    │   └── ...
    ├── create.yaml
    └── ...
```

**New Structure:**
```
parallelstore
├── __init__.py
└── instances
    ├── __init__.py
    ├── _partials/
    │   └── ...
    ├── create.yaml
    └── ...
```

Fixes #3543

Author: quirogas

internal/surfer/gcloud/generate.go

@@ -15,15 +15,18 @@
 package gcloud
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
+	"text/template"
 
 	"github.com/googleapis/librarian/internal/sidekick/api"
 	"github.com/googleapis/librarian/internal/surfer/gcloud/utils"
 	"github.com/googleapis/librarian/internal/yaml"
+	"github.com/iancoleman/strcase"
 )
 
 // partialsHeader is the directive that tells gcloud to look in the `_partials` directory
@@ -69,6 +72,21 @@ func generateService(service *api.Service, overrides *Config, model *api.API, ou
 	// `{outdir}/{shortServiceName}/`
 	surfaceDir := filepath.Join(output, shortServiceName)
 
+	if err := os.MkdirAll(surfaceDir, 0755); err != nil {
+		return fmt.Errorf("failed to create surface directory for %q: %w", shortServiceName, err)
+	}
+
+	track := strings.ToUpper(utils.InferTrackFromPackage(service.Package))
+	data := commandGroupData{
+		ServiceTitle:    utils.GetServiceTitle(model, shortServiceName),
+		ClassNamePrefix: strcase.ToCamel(shortServiceName),
+		Tracks:          []string{track},
+	}
+
+	if err := writeCommandGroupFile(surfaceDir, data); err != nil {
+		return fmt.Errorf("failed to write command group file for service %q: %w", shortServiceName, err)
+	}
+
 	// gcloud commands are resource-centric commands (e.g., `gcloud parallelstore instances create`),
 	// so we first need to group all the API methods by the resource they operate on.
 	// We'll create a map where the key is the resource's collection ID (e.g., "instances")
@@ -106,10 +124,35 @@ func generateService(service *api.Service, overrides *Config, model *api.API, ou
 // For a given collectionID like "instances", this function will create a directory
 // `instances/` and populate it with `create.yaml`, `delete.yaml`, etc.
 func generateResourceCommands(collectionID string, methods []*api.Method, baseDir string, overrides *Config, model *api.API, service *api.Service) error {
+	if len(methods) == 0 {
+		return nil
+	}
+
 	// The main directory for the resource is named after its collection ID.
 	// Example: `{baseDir}/instances`
 	resourceDir := filepath.Join(baseDir, collectionID)
 
+	if err := os.MkdirAll(resourceDir, 0755); err != nil {
+		return fmt.Errorf("failed to create resource directory for %q: %w", collectionID, err)
+	}
+
+	singular := utils.GetSingularResourceNameForMethod(methods[0], model)
+
+	// We determine the short service name from the default host to use as a fallback title.
+	shortServiceName, _, _ := strings.Cut(service.DefaultHost, ".")
+
+	track := strings.ToUpper(utils.InferTrackFromPackage(service.Package))
+	data := commandGroupData{
+		ServiceTitle:     utils.GetServiceTitle(model, shortServiceName),
+		ResourceSingular: singular,
+		ClassNamePrefix:  strcase.ToCamel(collectionID),
+		Tracks:           []string{track},
+	}
+
+	if err := writeCommandGroupFile(resourceDir, data); err != nil {
+		return fmt.Errorf("failed to write command group file for resource %q: %w", collectionID, err)
+	}
+
 	// Gcloud commands are defined in a `_partials` directory. This allows
 	// for sharing command definitions across different release tracks (GA, Beta, Alpha).
 	partialsDir := filepath.Join(resourceDir, "_partials")
@@ -165,3 +208,36 @@ func generateResourceCommands(collectionID string, methods []*api.Method, baseDi
 	}
 	return nil
 }
+
+func writeCommandGroupFile(dir string, data commandGroupData) error {
+	var buf bytes.Buffer
+	if err := commandGroupTemplate.Execute(&buf, data); err != nil {
+		return err
+	}
+	path := filepath.Join(dir, "__init__.py")
+	return os.WriteFile(path, buf.Bytes(), 0644)
+}
+
+var commandGroupTemplate = template.Must(template.New("__init__.py").Funcs(template.FuncMap{
+	"toCamel": strcase.ToCamel,
+}).Parse(`# NOTE: This file is autogenerated and should not be edited by hand.
+"""Manage {{.ServiceTitle}}{{if .ResourceSingular}} {{.ResourceSingular}}{{end}} resources."""
+
+from googlecloudsdk.calliope import base
+
+
+{{range .Tracks}}@base.ReleaseTracks(base.ReleaseTrack.{{.}})
+@base.Autogenerated
+@base.Hidden
+class {{$.ClassNamePrefix}}{{. | toCamel}}(base.Group):
+  """Manage {{$.ServiceTitle}}{{if $.ResourceSingular}} {{$.ResourceSingular}}{{end}} resources."""
+
+
+{{end}}`))
+
+type commandGroupData struct {
+	ServiceTitle     string
+	ResourceSingular string
+	ClassNamePrefix  string
+	Tracks           []string
+}

internal/surfer/gcloud/generate_test.go

@@ -15,6 +15,7 @@
 package gcloud
 
 import (
+	"bytes"
 	"os"
 	"path/filepath"
 	"testing"
@@ -48,7 +49,7 @@ func TestGenerateService(t *testing.T) {
 				},
 			},
 			model: &api.API{
-				ResourceDefinitions: []*api.Resource{},
+				Title: "Parallelstore API",
 			},
 			wantErr: false,
 		},
@@ -57,8 +58,11 @@ func TestGenerateService(t *testing.T) {
 			service: &api.Service{
 				Name:        "parallelstore.googleapis.com",
 				DefaultHost: "",
+				Package:     "google.cloud.parallelstore.v1",
+			},
+			model: &api.API{
+				Title: "Parallelstore API",
 			},
-			model:   &api.API{},
 			wantErr: true,
 		},
 	} {
@@ -79,11 +83,13 @@ func TestGenerateResourceCommands(t *testing.T) {
 
 	err := generateResourceCommands("instances", []*api.Method{
 		{
-			Name:      "CreateInstance",
-			Service:   &api.Service{Package: "google.cloud.parallelstore.v1"},
+			Name: "CreateInstance",
+			Service: &api.Service{
+				Package: "google.cloud.parallelstore.v1",
+			},
 			InputType: &api.Message{},
 		},
-	}, tmpDir, &Config{}, &api.API{}, &api.Service{DefaultHost: "parallelstore.googleapis.com"})
+	}, tmpDir, &Config{}, &api.API{Title: "Parallelstore API"}, &api.Service{DefaultHost: "parallelstore.googleapis.com"})
 
 	if err != nil {
 		t.Fatalf("generateResourceCommands() error = %v", err)
@@ -101,4 +107,19 @@ func TestGenerateResourceCommands(t *testing.T) {
 	if diff := cmp.Diff(wantContent, string(content)); diff != "" {
 		t.Errorf("main file content mismatch (-want +got):\n%s", diff)
 	}
+
+	// Check __init__.py content
+	initFile := filepath.Join(tmpDir, "instances", "__init__.py")
+	if _, err := os.Stat(initFile); os.IsNotExist(err) {
+		t.Errorf("expected file %s to exist", initFile)
+	}
+	initContent, _ := os.ReadFile(initFile)
+	// The mock model is empty, so we expect some defaults.
+	// But let's see if it matches the general structure.
+	if !bytes.Contains(initContent, []byte(`"""Manage Parallelstore resources."""`)) {
+		t.Errorf("__init__.py content missing expected docstring:\n%s", string(initContent))
+	}
+	if !bytes.Contains(initContent, []byte("class InstancesGa(base.Group):")) {
+		t.Errorf("__init__.py content missing expected class definition:\n%s", string(initContent))
+	}
 }

internal/surfer/gcloud/utils/resource.go

@@ -195,3 +195,19 @@ func GetPluralResourceNameForMethod(method *api.Method, model *api.API) string {
 	}
 	return ""
 }
+
+// GetSingularResourceNameForMethod determines the singular name of a resource. It follows a clear
+// hierarchy of truth: first, the explicit `singular` field in the resource
+// definition, and second, inference from the resource pattern.
+func GetSingularResourceNameForMethod(method *api.Method, model *api.API) string {
+	resource := GetResourceForMethod(method, model)
+	if resource != nil {
+		if resource.Singular != "" {
+			return resource.Singular
+		}
+		if len(resource.Patterns) > 0 {
+			return GetSingularFromSegments(resource.Patterns[0])
+		}
+	}
+	return ""
+}

internal/surfer/gcloud/utils/resource_test.go

@@ -446,3 +446,77 @@ func TestGetPluralResourceNameForMethod(t *testing.T) {
 		})
 	}
 }
+
+func TestGetSingularResourceNameForMethod(t *testing.T) {
+	instanceResource := &api.Resource{
+		Type: "example.googleapis.com/Instance",
+		Patterns: []api.ResourcePattern{
+			{
+				*api.NewPathSegment().WithLiteral("instances"),
+				*api.NewPathSegment().WithVariable(api.NewPathVariable("instance").WithMatch()),
+			},
+		},
+	}
+	model := &api.API{
+		ResourceDefinitions: []*api.Resource{
+			instanceResource,
+		},
+	}
+
+	for _, test := range []struct {
+		name   string
+		method *api.Method
+		want   string
+	}{
+		{
+			name: "Inferred from Pattern",
+			method: &api.Method{
+				Name: "ListInstances",
+				InputType: &api.Message{
+					Fields: []*api.Field{
+						{
+							Name: "parent",
+							ResourceReference: &api.ResourceReference{
+								ChildType: "example.googleapis.com/Instance",
+							},
+						},
+					},
+				},
+			},
+			want: "instance",
+		},
+		{
+			name: "Explicit Singular",
+			method: &api.Method{
+				Name: "ListBooks",
+				InputType: &api.Message{
+					Fields: []*api.Field{
+						{
+							Name: "parent",
+							ResourceReference: &api.ResourceReference{
+								ChildType: "example.googleapis.com/Book",
+							},
+						},
+					},
+				},
+			},
+			want: "book", // Assuming we mock a Book resource with Singular="book" below
+		},
+	} {
+		// Setup explicit singular for the second case
+		if test.name == "Explicit Singular" {
+			bookResource := &api.Resource{
+				Type:     "example.googleapis.com/Book",
+				Singular: "book",
+			}
+			model.ResourceDefinitions = append(model.ResourceDefinitions, bookResource)
+		}
+
+		t.Run(test.name, func(t *testing.T) {
+			got := GetSingularResourceNameForMethod(test.method, model)
+			if diff := cmp.Diff(test.want, got); diff != "" {
+				t.Errorf("GetSingularResourceNameForMethod mismatch (-want +got):\n%s", diff)
+			}
+		})
+	}
+}

internal/surfer/gcloud/utils/service.go

@@ -14,7 +14,12 @@
 
 package utils
 
-import "strings"
+import (
+	"strings"
+
+	"github.com/googleapis/librarian/internal/sidekick/api"
+	"github.com/iancoleman/strcase"
+)
 
 // InferTrackFromPackage infers the release track from the proto package name.
 // as mandated per AIP-185
@@ -36,3 +41,12 @@ func InferTrackFromPackage(pkg string) string {
 	}
 	return "ga"
 }
+
+// GetServiceTitle returns the service title for documentation.
+// It tries to use the API title, falling back to a CamelCase version of the short service name.
+func GetServiceTitle(model *api.API, shortServiceName string) string {
+	if t := strings.TrimSuffix(model.Title, " API"); t != "" {
+		return t
+	}
+	return strcase.ToCamel(shortServiceName)
+}

internal/surfer/gcloud/utils/service_test.go

@@ -18,6 +18,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/googleapis/librarian/internal/sidekick/api"
 )
 
 func TestInferTrackFromPackage(t *testing.T) {
@@ -41,3 +42,52 @@ func TestInferTrackFromPackage(t *testing.T) {
 		})
 	}
 }
+
+func TestGetServiceTitle(t *testing.T) {
+	for _, test := range []struct {
+		name             string
+		model            *api.API
+		shortServiceName string
+		want             string
+	}{
+		{
+			name: "With API Suffix",
+			model: &api.API{
+				Title: "Parallelstore API",
+			},
+			shortServiceName: "parallelstore",
+			want:             "Parallelstore",
+		},
+		{
+			name: "Without API Suffix",
+			model: &api.API{
+				Title: "Parallelstore",
+			},
+			shortServiceName: "parallelstore",
+			want:             "Parallelstore",
+		},
+		{
+			name: "Empty Title",
+			model: &api.API{
+				Title: "",
+			},
+			shortServiceName: "parallelstore",
+			want:             "Parallelstore",
+		},
+		{
+			name: "Empty Title and different short name",
+			model: &api.API{
+				Title: "",
+			},
+			shortServiceName: "cloudbuild",
+			want:             "Cloudbuild",
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			got := GetServiceTitle(test.model, test.shortServiceName)
+			if diff := cmp.Diff(test.want, got); diff != "" {
+				t.Errorf("mismatch (-want +got):\n%s", diff)
+			}
+		})
+	}
+}