Skip to content
This repository was archived by the owner on Mar 26, 2026. It is now read-only.

Commit 4b1c823

Browse files
authored
Dual templates (#357)
Add a parallel set of templates, used primarily for generation of https://github.com/googleads/google-ads-python/ Parameterize nox tests to take template path configuration Run pytest tests in parallel from nox
1 parent 5990e53 commit 4b1c823

33 files changed

Lines changed: 2947 additions & 43 deletions

.circleci/config.yml

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,42 @@ workflows:
4444
filters:
4545
tags:
4646
only: /^\d+\.\d+\.\d+$/
47+
- showcase-unit-alternative-templates-3.6:
48+
requires:
49+
- unit-3.6
50+
- unit-3.7
51+
- unit-3.8
52+
filters:
53+
tags:
54+
only: /^\d+\.\d+\.\d+$/
55+
- showcase-unit-alternative-templates-3.7:
56+
requires:
57+
- unit-3.6
58+
- unit-3.7
59+
- unit-3.8
60+
filters:
61+
tags:
62+
only: /^\d+\.\d+\.\d+$/
63+
- showcase-unit-alternative-templates-3.8:
64+
requires:
65+
- unit-3.6
66+
- unit-3.7
67+
- unit-3.8
68+
filters:
69+
tags:
70+
only: /^\d+\.\d+\.\d+$/
4771
- showcase-mypy:
4872
requires:
4973
- mypy
5074
filters:
5175
tags:
5276
only: /^\d+\.\d+\.\d+$/
77+
- showcase-mypy-alternative-templates:
78+
requires:
79+
- mypy
80+
filters:
81+
tags:
82+
only: /^\d+\.\d+\.\d+$/
5383
- showcase:
5484
requires:
5585
- docs
@@ -61,6 +91,17 @@ workflows:
6191
filters:
6292
tags:
6393
only: /^\d+\.\d+\.\d+$/
94+
- showcase-alternative-templates:
95+
requires:
96+
- docs
97+
- mypy
98+
- showcase-unit-alternative-templates-3.6
99+
- showcase-unit-alternative-templates-3.7
100+
- showcase-unit-alternative-templates-3.8
101+
- showcase-mypy-alternative-templates
102+
filters:
103+
tags:
104+
only: /^\d+\.\d+\.\d+$/
64105
- docs:
65106
filters:
66107
tags:
@@ -188,6 +229,31 @@ jobs:
188229
- run:
189230
name: Run showcase tests.
190231
command: nox -s showcase
232+
showcase-alternative-templates:
233+
docker:
234+
- image: python:3.8-slim
235+
- image: gcr.io/gapic-images/gapic-showcase:0.6.1
236+
steps:
237+
- checkout
238+
- run:
239+
name: Install system dependencies.
240+
command: |
241+
apt-get update
242+
apt-get install -y curl pandoc unzip
243+
- run:
244+
name: Install nox.
245+
command: pip install nox
246+
- run:
247+
name: Install protoc 3.7.1.
248+
command: |
249+
mkdir -p /usr/src/protoc/
250+
curl --location https://github.com/google/protobuf/releases/download/v3.7.1/protoc-3.7.1-linux-x86_64.zip --output /usr/src/protoc/protoc-3.7.1.zip
251+
cd /usr/src/protoc/
252+
unzip protoc-3.7.1.zip
253+
ln -s /usr/src/protoc/bin/protoc /usr/local/bin/protoc
254+
- run:
255+
name: Run showcase tests.
256+
command: nox -s showcase_alternative_templates
191257
showcase-unit-3.6:
192258
docker:
193259
- image: python:3.6-slim
@@ -260,6 +326,78 @@ jobs:
260326
- run:
261327
name: Run unit tests.
262328
command: nox -s showcase_unit-3.8
329+
showcase-unit-alternative-templates-3.6:
330+
docker:
331+
- image: python:3.6-slim
332+
steps:
333+
- checkout
334+
- run:
335+
name: Install system dependencies.
336+
command: |
337+
apt-get update
338+
apt-get install -y curl pandoc unzip
339+
- run:
340+
name: Install protoc 3.7.1.
341+
command: |
342+
mkdir -p /usr/src/protoc/
343+
curl --location https://github.com/google/protobuf/releases/download/v3.7.1/protoc-3.7.1-linux-x86_64.zip --output /usr/src/protoc/protoc-3.7.1.zip
344+
cd /usr/src/protoc/
345+
unzip protoc-3.7.1.zip
346+
ln -s /usr/src/protoc/bin/protoc /usr/local/bin/protoc
347+
- run:
348+
name: Install nox.
349+
command: pip install nox
350+
- run:
351+
name: Run unit tests.
352+
command: nox -s showcase_unit_alternative_templates-3.6
353+
showcase-unit-alternative-templates-3.7:
354+
docker:
355+
- image: python:3.7-slim
356+
steps:
357+
- checkout
358+
- run:
359+
name: Install system dependencies.
360+
command: |
361+
apt-get update
362+
apt-get install -y curl pandoc unzip
363+
- run:
364+
name: Install protoc 3.7.1.
365+
command: |
366+
mkdir -p /usr/src/protoc/
367+
curl --location https://github.com/google/protobuf/releases/download/v3.7.1/protoc-3.7.1-linux-x86_64.zip --output /usr/src/protoc/protoc-3.7.1.zip
368+
cd /usr/src/protoc/
369+
unzip protoc-3.7.1.zip
370+
ln -s /usr/src/protoc/bin/protoc /usr/local/bin/protoc
371+
- run:
372+
name: Install nox.
373+
command: pip install nox
374+
- run:
375+
name: Run unit tests.
376+
command: nox -s showcase_unit_alternative_templates-3.7
377+
showcase-unit-alternative-templates-3.8:
378+
docker:
379+
- image: python:3.8-slim
380+
steps:
381+
- checkout
382+
- run:
383+
name: Install system dependencies.
384+
command: |
385+
apt-get update
386+
apt-get install -y curl pandoc unzip
387+
- run:
388+
name: Install protoc 3.7.1.
389+
command: |
390+
mkdir -p /usr/src/protoc/
391+
curl --location https://github.com/google/protobuf/releases/download/v3.7.1/protoc-3.7.1-linux-x86_64.zip --output /usr/src/protoc/protoc-3.7.1.zip
392+
cd /usr/src/protoc/
393+
unzip protoc-3.7.1.zip
394+
ln -s /usr/src/protoc/bin/protoc /usr/local/bin/protoc
395+
- run:
396+
name: Install nox.
397+
command: pip install nox
398+
- run:
399+
name: Run unit tests.
400+
command: nox -s showcase_unit_alternative_templates-3.8
263401
showcase-mypy:
264402
docker:
265403
- image: python:3.8-slim
@@ -284,6 +422,30 @@ jobs:
284422
- run:
285423
name: Typecheck the generated output.
286424
command: nox -s showcase_mypy
425+
showcase-mypy-alternative-templates:
426+
docker:
427+
- image: python:3.8-slim
428+
steps:
429+
- checkout
430+
- run:
431+
name: Install system dependencies.
432+
command: |
433+
apt-get update
434+
apt-get install -y curl pandoc unzip
435+
- run:
436+
name: Install protoc 3.7.1.
437+
command: |
438+
mkdir -p /usr/src/protoc/
439+
curl --location https://github.com/google/protobuf/releases/download/v3.7.1/protoc-3.7.1-linux-x86_64.zip --output /usr/src/protoc/protoc-3.7.1.zip
440+
cd /usr/src/protoc/
441+
unzip protoc-3.7.1.zip
442+
ln -s /usr/src/protoc/bin/protoc /usr/local/bin/protoc
443+
- run:
444+
name: Install nox.
445+
command: pip install nox
446+
- run:
447+
name: Typecheck the generated output.
448+
command: nox -s showcase_mypy_alternative_templates
287449
unit-3.6:
288450
docker:
289451
- image: python:3.6-slim
Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
{% extends '_base.py.j2' %}
2+
3+
{% block content %}
4+
{% if opts.lazy_import -%} {# lazy import #}
5+
import importlib
6+
import re
7+
import sys
8+
9+
from itertools import chain
10+
11+
def to_snake_case(s: str) -> str:
12+
s = re.sub(r'(?<=[a-z])([A-Z])', r'_\1', str(s))
13+
s = re.sub(r'(?<=[^_])([A-Z])(?=[a-z])', r'_\1', s)
14+
15+
# Numbers are a weird case; the goal is to spot when they _start_
16+
# some kind of name or acronym (e.g. 2FA, 3M).
17+
#
18+
# Find cases of a number preceded by a lower-case letter _and_
19+
# followed by at least two capital letters or a single capital and
20+
# end of string.
21+
s = re.sub(r'(?<=[a-z])(\d)(?=[A-Z]{2})', r'_\1', s)
22+
s = re.sub(r'(?<=[a-z])(\d)(?=[A-Z]$)', r'_\1', s)
23+
24+
return s.lower()
25+
26+
27+
def from_snake_case(s):
28+
_CHARS_TO_UPCASE_RE = re.compile(r'(?:_|^)([a-z])')
29+
return _CHARS_TO_UPCASE_RE.sub(lambda m: m.group().replace('_', '').upper(), s)
30+
31+
32+
if sys.version_info < (3, 7):
33+
raise ImportError('This module requires Python 3.7 or later.') # pragma: NO COVER
34+
35+
_lazy_name_to_package_map = {
36+
'types': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.types',
37+
{%- for service in api.services.values()|sort(attribute='name')|unique(attribute='name') if service.meta.address.subpackage == api.subpackage_view %}
38+
'{{ service.client_name|snake_case }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.services.{{ service.name|snake_case }}.client',
39+
'{{ service.transport_name|snake_case }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.services.{{ service.name|snake_case }}.transports.base',
40+
'{{ service.grpc_transport_name|snake_case }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.services.{{ service.name|snake_case }}.transports.grpc',
41+
{%- endfor %}
42+
}
43+
44+
_lazy_type_to_package_map = {
45+
{%- filter sort_lines %}
46+
{%- for proto in api.protos.values() if proto.meta.address.subpackage == api.subpackage_view %}{%- for message in proto.messages.values() %}
47+
'{{ message.name }}':'{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.types.{{ proto.module_name }}',
48+
{%- endfor %}
49+
{%- for enum in proto.enums.values() %}
50+
'{{ enum.name }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.types.{{ proto.module_name }}',
51+
{%- endfor %}{%- endfor %}{%- endfilter %}
52+
}
53+
54+
# Background on how this behaves: https://www.python.org/dev/peps/pep-0562/
55+
def __getattr__(name): # Requires Python >= 3.7
56+
if name == '__all__':
57+
all_names = globals()['__all__'] = sorted(
58+
chain(
59+
(from_snake_case(k) for k in _lazy_name_to_package_map if k != 'types'),
60+
_lazy_type_to_package_map,
61+
['types'],
62+
)
63+
)
64+
return all_names
65+
elif name.endswith('Transport'):
66+
module = __getattr__(to_snake_case(name))
67+
sub_mod_class = getattr(module, name)
68+
klass = type(name, (sub_mod_class,), {'__doc__': sub_mod_class.__doc__})
69+
globals()[name] = klass
70+
return klass
71+
elif name.endswith('Client'):
72+
module = __getattr__(to_snake_case(name))
73+
sub_mod_class = getattr(module, name)
74+
klass = type(
75+
name,
76+
(sub_mod_class,),
77+
{'__doc__': sub_mod_class.__doc__}
78+
)
79+
globals()[name] = klass
80+
return klass
81+
elif name in _lazy_name_to_package_map:
82+
module = importlib.import_module(f'{_lazy_name_to_package_map[name]}')
83+
globals()[name] = module
84+
return module
85+
elif name in _lazy_type_to_package_map:
86+
module = importlib.import_module(f'{_lazy_type_to_package_map[name]}')
87+
klass = getattr(module, name)
88+
{# new_klass = type(name, (klass,), {'__doc__': klass.__doc__}) #}
89+
globals()[name] = klass
90+
return klass
91+
else:
92+
raise AttributeError(f'unknown sub-module {name!r}.')
93+
94+
95+
def __dir__():
96+
return globals().get('__all__') or __getattr__('__all__')
97+
{% else -%} {# do not use lazy import #}
98+
{# Import subpackages. -#}
99+
{% for subpackage in api.subpackages.keys() -%}
100+
from . import {{ subpackage }}
101+
{% endfor -%}
102+
103+
{# Import services for this package. -#}
104+
{% filter sort_lines -%}
105+
{% for service in api.services.values()|sort(attribute='name')
106+
if service.meta.address.subpackage == api.subpackage_view -%}
107+
from .services.{{ service.name|snake_case }} import {{ service.client_name }}
108+
{% endfor -%}
109+
{% endfilter -%}
110+
111+
{# Import messages and enums from each proto.
112+
It is safe to import all of the messages into the same namespace here,
113+
because protocol buffers itself enforces selector uniqueness within
114+
a proto package.
115+
-#}
116+
{% filter sort_lines -%}
117+
{% for proto in api.protos.values()
118+
if proto.meta.address.subpackage == api.subpackage_view -%}
119+
{% for message in proto.messages.values() -%}
120+
from .types.{{ proto.module_name }} import {{ message.name }}
121+
{% endfor -%}
122+
{% for enum in proto.enums.values() -%}
123+
from .types.{{ proto.module_name }} import {{ enum.name }}
124+
{% endfor -%}
125+
{% endfor -%}
126+
{% endfilter %}
127+
128+
{# Define __all__.
129+
This requires the full set of imported names, so we iterate over
130+
them again.
131+
-#}
132+
__all__ = (
133+
{%- filter sort_lines %}
134+
{%- for subpackage in api.subpackages.keys() %}
135+
'{{ subpackage }}',
136+
{%- endfor %}
137+
{%- for service in api.services.values()
138+
if service.meta.address.subpackage == api.subpackage_view %}
139+
'{{ service.client_name }}',
140+
{%- endfor %}
141+
{%- for proto in api.protos.values()
142+
if proto.meta.address.subpackage == api.subpackage_view %}
143+
{%- for message in proto.messages.values() %}
144+
'{{ message.name }}',
145+
{%- endfor %}
146+
{%- for enum in proto.enums.values() %}
147+
'{{ enum.name }}',
148+
{%- endfor %}
149+
{%- endfor %}
150+
{%- endfilter %}
151+
)
152+
{% endif -%} {# lazy import #}
153+
{% endblock %}
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{% extends '_base.py.j2' %}
2+
3+
{% block content %}
4+
from .client import {{ service.client_name }}
5+
6+
__all__ = (
7+
'{{ service.client_name }}',
8+
)
9+
{% endblock %}

0 commit comments

Comments
 (0)