diff --git a/packages/gapic-generator/gapic/generator/generator.py b/packages/gapic-generator/gapic/generator/generator.py index 9fe56aa9de1d..c1932abbe426 100644 --- a/packages/gapic-generator/gapic/generator/generator.py +++ b/packages/gapic-generator/gapic/generator/generator.py @@ -120,7 +120,7 @@ def get_response(self, api_schema: api.API, opts: Options) -> CodeGeneratorRespo for template_name in client_templates: # Quick check: Skip "private" templates. filename = template_name.split("/")[-1] - if filename.startswith("_") and filename != "__init__.py.j2": + if filename.startswith("_") and filename not in ("__init__.py.j2", "_compat.py.j2"): continue # Append to the output files dictionary. diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/_compat.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/_compat.py.j2 new file mode 100644 index 000000000000..753dd6026676 --- /dev/null +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/_compat.py.j2 @@ -0,0 +1,96 @@ +# {% include '_license.j2' %} + +from typing import Any, Dict, List, Optional, Tuple + +try: + from google.api_core.rest_helpers import ( + flatten_query_params, + transcode_request, + ) +except ImportError: # pragma: NO COVER + # TODO: Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + import functools + import json + import operator + from google.protobuf import json_format # type: ignore + from google.api_core import path_template # type: ignore + + def flatten_query_params(obj, strict=False): # pragma: NO COVER + if obj is not None and not isinstance(obj, dict): + raise TypeError("flatten_query_params must be called with dict object") + return _flatten(obj, key_path=[], strict=strict) + + def _flatten(obj, key_path, strict=False): # pragma: NO COVER + if obj is None: + return [] + if isinstance(obj, dict): + return _flatten_dict(obj, key_path=key_path, strict=strict) + if isinstance(obj, list): + return _flatten_list(obj, key_path=key_path, strict=strict) + return _flatten_value(obj, key_path=key_path, strict=strict) + + def _is_primitive_value(obj): # pragma: NO COVER + if obj is None: + return False + if isinstance(obj, (list, dict)): + raise ValueError("query params may not contain repeated dicts or lists") + return True + + def _flatten_value(obj, key_path, strict=False): # pragma: NO COVER + return [(".".join(key_path), _canonicalize(obj, strict=strict))] + + def _flatten_dict(obj, key_path, strict=False): # pragma: NO COVER + items = ( + _flatten(value, key_path=key_path + [key], strict=strict) + for key, value in obj.items() + ) + return functools.reduce(operator.concat, items, []) + + def _flatten_list(elems, key_path, strict=False): # pragma: NO COVER + items = ( + _flatten_value(elem, key_path=key_path, strict=strict) + for elem in elems + if _is_primitive_value(elem) + ) + return functools.reduce(operator.concat, items, []) + + def _canonicalize(obj, strict=False): # pragma: NO COVER + if strict: + value = str(obj) + if isinstance(obj, bool): + value = value.lower() + return value + return obj + + def transcode_request( # pragma: NO COVER + http_options: List[Dict[str, str]], + request: Any, + required_fields_default_values: Optional[Dict[str, Any]] = None, + rest_numeric_enums: bool = False, + ) -> Tuple[Dict[str, Any], Optional[str], Dict[str, Any]]: + pb_request = getattr(request, "_pb", request) + transcoded_request = path_template.transcode(http_options, pb_request) + + body_json = None + if transcoded_request.get("body") is not None: + body_json = json_format.MessageToJson( + transcoded_request["body"], + use_integers_for_enums=rest_numeric_enums, + ) + + query_params_json = {} + if transcoded_request.get("query_params") is not None: + query_params_json = json.loads(json_format.MessageToJson( + transcoded_request["query_params"], + use_integers_for_enums=rest_numeric_enums, + )) + + if required_fields_default_values: + for k, v in required_fields_default_values.items(): + if k not in query_params_json: + query_params_json[k] = v + + if rest_numeric_enums: + query_params_json["$alt"] = "json;enum-encoding=int" + + return transcoded_request, body_json, query_params_json diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2 index 755e4530e7ba..6e58759112ca 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2 @@ -198,22 +198,23 @@ def _get_http_options(): service: The service. is_async (bool): Used to determine the code path i.e. whether for sync or async call. is_request_message_proto_plus_type (bool): Used to determine whether the request message is a proto-plus type. #} -{% macro rest_call_method_common(body_spec, method_name, service, is_async=False, is_request_message_proto_plus_type=False) %} +{% macro rest_call_method_common(body_spec, method_name, service, is_async=False, is_request_message_proto_plus_type=False, rest_numeric_enums=False) %} {% set service_name = service.name %} {% set await_prefix = "await " if is_async else "" %} {% set async_class_prefix = "Async" if is_async else "" %} http_options = _Base{{ service_name }}RestTransport._Base{{method_name}}._get_http_options() - {# TODO(https://github.com/googleapis/gapic-generator-python/issues/2274): Add debug log before intercepting a request #} request, metadata = {{ await_prefix }}self._interceptor.pre_{{ method_name|snake_case }}(request, metadata) - transcoded_request = _Base{{ service_name }}RestTransport._Base{{method_name}}._get_transcoded_request(http_options, request) - - {% if body_spec %} - body = _Base{{ service_name }}RestTransport._Base{{method_name}}._get_request_body_json(transcoded_request) - {% endif %}{# body_spec #} - - # Jsonify the query params - query_params = _Base{{ service_name }}RestTransport._Base{{method_name}}._get_query_params_json(transcoded_request) + transcoded_request, body, query_params = rest_helpers.transcode_request( + http_options, + request, + required_fields_default_values=getattr( + _Base{{ service_name }}RestTransport._Base{{method_name}}, + "__REQUIRED_FIELDS_DEFAULT_VALUES", + None, + ), + rest_numeric_enums={{ rest_numeric_enums }}, + ) if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor(logging.DEBUG): # pragma: NO COVER request_url = "{host}{uri}".format(host=self._host, uri=transcoded_request['uri']) diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/_rest_mixins_base.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/_rest_mixins_base.py.j2 index 16cc77ea937c..f768500d5755 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/_rest_mixins_base.py.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/_rest_mixins_base.py.j2 @@ -25,27 +25,7 @@ {{ shared_macros.http_options_method(api.mixin_http_options["{}".format(name)])|indent(8)}} - @staticmethod - def _get_transcoded_request(http_options, request): - request_kwargs = json_format.MessageToDict(request) - transcoded_request = path_template.transcode( - http_options, **request_kwargs) - return transcoded_request - - {% set body_spec = api.mixin_http_options["{}".format(name)][0].body %} - {%- if body_spec %} - - @staticmethod - def _get_request_body_json(transcoded_request): - body = json.dumps(transcoded_request['body']) - return body - - {%- endif %} {# body_spec #} - - @staticmethod - def _get_query_params_json(transcoded_request): - query_params = json.loads(json.dumps(transcoded_request['query_params'])) - return query_params + pass {% endfor %} {% endif %} {# rest in opts.transport #} diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest.py.j2 index 49c1374053b5..a3e08a447d06 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest.py.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest.py.j2 @@ -10,7 +10,8 @@ from google.auth.transport.requests import AuthorizedSession # type: ignore from google.auth import credentials as ga_credentials # type: ignore from google.api_core import exceptions as core_exceptions from google.api_core import retry as retries -from google.api_core import rest_helpers +{% set package_path = api.naming.module_namespace|join('.') + "." + api.naming.versioned_module_name %} +from {{package_path}} import _compat as rest_helpers from google.api_core import rest_streaming from google.api_core import gapic_v1 import google.protobuf @@ -245,7 +246,7 @@ class {{service.name}}RestTransport(_Base{{ service.name }}RestTransport): {% endif %} """ - {{ shared_macros.rest_call_method_common(body_spec, method.name, service, False, method.input.ident.is_proto_plus_type)|indent(8) }} + {{ shared_macros.rest_call_method_common(body_spec, method.name, service, False, method.input.ident.is_proto_plus_type, opts.rest_numeric_enums)|indent(8) }} {% if not method.void %} # Return the response diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_asyncio.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_asyncio.py.j2 index 80980572c30a..83c8cd28b986 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_asyncio.py.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_asyncio.py.j2 @@ -32,7 +32,8 @@ from google.iam.v1 import policy_pb2 # type: ignore from google.cloud.location import locations_pb2 # type: ignore {% endif %} from google.api_core import retry_async as retries -from google.api_core import rest_helpers +{% set package_path = api.naming.module_namespace|join('.') + "." + api.naming.versioned_module_name %} +from {{package_path}} import _compat as rest_helpers from google.api_core import rest_streaming_async # type: ignore import google.protobuf @@ -203,7 +204,7 @@ class Async{{service.name}}RestTransport(_Base{{ service.name }}RestTransport): {% endif %} """ - {{ shared_macros.rest_call_method_common(body_spec, method.name, service, True, method.input.ident.is_proto_plus_type)|indent(8) }} + {{ shared_macros.rest_call_method_common(body_spec, method.name, service, True, method.input.ident.is_proto_plus_type, opts.rest_numeric_enums)|indent(8) }} {% if not method.void %} # Return the response diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_base.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_base.py.j2 index b79785afc517..90c0e10d6403 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_base.py.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_base.py.j2 @@ -120,51 +120,8 @@ class _Base{{ service.name }}RestTransport({{service.name}}Transport): def _get_unset_required_fields(cls, message_dict): return {k: v for k, v in cls.__REQUIRED_FIELDS_DEFAULT_VALUES.items() if k not in message_dict} {% endif %}{# required fields #} - {% set method_http_options = method.http_options %} - {{ shared_macros.http_options_method(method_http_options)|indent(8) }} - - @staticmethod - def _get_transcoded_request(http_options, request): - {% if method.input.ident.is_proto_plus_type %} - pb_request = {{method.input.ident}}.pb(request) - {% else %} - pb_request = request - {% endif %} - transcoded_request = path_template.transcode(http_options, pb_request) - return transcoded_request - - {% set body_spec = method.http_options[0].body %} - {%- if body_spec %} - - @staticmethod - def _get_request_body_json(transcoded_request): - # Jsonify the request body - - body = json_format.MessageToJson( - transcoded_request['body'], - use_integers_for_enums={{ opts.rest_numeric_enums }} - ) - return body - - {%- endif %}{# body_spec #} - - @staticmethod - def _get_query_params_json(transcoded_request): - query_params = json.loads(json_format.MessageToJson( - transcoded_request['query_params'], - use_integers_for_enums={{ opts.rest_numeric_enums }}, - )) - {% if method.input.required_fields %} - query_params.update(_Base{{ service.name }}RestTransport._Base{{method.name}}._get_unset_required_fields(query_params)) - {% endif %}{# required fields #} - - {% if opts.rest_numeric_enums %} - query_params["$alt"] = "json;enum-encoding=int" - {% endif %} - return query_params - {% endif %}{# method.http_options and not method.client_streaming #} {% endfor %} diff --git a/packages/gapic-generator/noxfile.py b/packages/gapic-generator/noxfile.py index 52209f41ff38..d56fed3e20f7 100644 --- a/packages/gapic-generator/noxfile.py +++ b/packages/gapic-generator/noxfile.py @@ -350,6 +350,7 @@ def showcase_library( ) # Install the generated showcase library. + session.install("-e", "../google-api-core") if templates == "DEFAULT": # Use the constraints file for the specific python runtime version. # We do this to make sure that we're testing against the lowest @@ -380,6 +381,9 @@ def showcase_library( # Install the library without a constraints file. session.install("-e", tmp_dir) + import shutil + shutil.rmtree("generated_showcase", ignore_errors=True) + shutil.copytree(tmp_dir, "generated_showcase") yield tmp_dir @@ -504,6 +508,10 @@ def run_showcase_unit_tests(session, fail_under=100, rest_async_io_enabled=False session.run("python", "-m", "pip", "freeze") # Run the tests. + import os + env = os.environ.copy() + env["GOOGLE_API_USE_CLIENT_CERTIFICATE"] = "false" + env["CLOUDSDK_CONTEXT_AWARE_USE_CLIENT_CERTIFICATE"] = "" session.run( "py.test", *( @@ -517,6 +525,7 @@ def run_showcase_unit_tests(session, fail_under=100, rest_async_io_enabled=False path.join("tests", "unit"), ] ), + env=env, ) diff --git a/packages/google-api-core/google/api_core/rest_helpers.py b/packages/google-api-core/google/api_core/rest_helpers.py index a78822f1572a..4d058fca33f7 100644 --- a/packages/google-api-core/google/api_core/rest_helpers.py +++ b/packages/google-api-core/google/api_core/rest_helpers.py @@ -16,6 +16,11 @@ import functools import operator +from typing import Any, Dict, List, Optional, Tuple + +from google.protobuf import json_format + +from google.api_core import path_template def flatten_query_params(obj, strict=False): @@ -107,3 +112,54 @@ def _canonicalize(obj, strict=False): value = value.lower() return value return obj + + +def transcode_request( + http_options: List[Dict[str, str]], + request: Any, + required_fields_default_values: Optional[Dict[str, Any]] = None, + rest_numeric_enums: bool = False, +) -> Tuple[Dict[str, Any], Optional[str], Dict[str, Any]]: + """Transcodes a request into HTTP method, URI, body, and query parameters. + + Args: + http_options (List[Dict[str, str]]): List of HTTP transcoding rules. + request (Any): The protobuf or proto-plus request message. + required_fields_default_values (Optional[Dict[str, Any]]): Dictionary + of required fields default values to merge into query parameters if missing. + rest_numeric_enums (bool): Whether to encode enums as integers. + + Returns: + Tuple[Dict[str, Any], Optional[str], Dict[str, Any]]: A tuple containing: + - The raw transcoded request dictionary (containing keys like 'uri', 'method'). + - The serialized request body JSON string, or None if no body. + - The query parameters dictionary. + """ + # Convert proto-plus message to its underlying protobuf message if needed + pb_request = getattr(request, "_pb", request) + + transcoded_request = path_template.transcode(http_options, pb_request) + + body_json = None + if transcoded_request.get("body") is not None: + body_json = json_format.MessageToJson( + transcoded_request["body"], + use_integers_for_enums=rest_numeric_enums, + ) + + query_params_json = {} + if transcoded_request.get("query_params") is not None: + query_params_json = json_format.MessageToDict( + transcoded_request["query_params"], + use_integers_for_enums=rest_numeric_enums, + ) + + if required_fields_default_values: + for k, v in required_fields_default_values.items(): + if k not in query_params_json: + query_params_json[k] = v + + if rest_numeric_enums: + query_params_json["$alt"] = "json;enum-encoding=int" + + return transcoded_request, body_json, query_params_json diff --git a/packages/google-api-core/tests/conftest.py b/packages/google-api-core/tests/conftest.py new file mode 100644 index 000000000000..62a3c999f733 --- /dev/null +++ b/packages/google-api-core/tests/conftest.py @@ -0,0 +1,31 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from unittest import mock + +import pytest + + +@pytest.fixture(scope="session", autouse=True) +def mock_mtls_env(): + """Autouse session-scoped fixture to isolate unit tests from workstation mTLS environments.""" + with mock.patch.dict( + os.environ, + { + "GOOGLE_API_USE_CLIENT_CERTIFICATE": "false", + "CLOUDSDK_CONTEXT_AWARE_USE_CLIENT_CERTIFICATE": "false", + }, + ): + yield diff --git a/packages/google-api-core/tests/unit/test_rest_helpers.py b/packages/google-api-core/tests/unit/test_rest_helpers.py index ff1a43f002aa..e0b9323c018b 100644 --- a/packages/google-api-core/tests/unit/test_rest_helpers.py +++ b/packages/google-api-core/tests/unit/test_rest_helpers.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import pytest +from google.protobuf import descriptor_pb2 from google.api_core import rest_helpers +from google.api_core.rest_helpers import transcode_request def test_flatten_simple_value(): @@ -92,3 +95,120 @@ def test_flatten_repeated_list(): with pytest.raises(ValueError): rest_helpers.flatten_query_params(obj) + + +def test_transcode_basic(): + # We use FieldDescriptorProto as it has standard primitive fields and nested messages. + http_options = [ + { + "method": "get", + "uri": "/v1/test/{name}", + } + ] + + request = descriptor_pb2.FieldDescriptorProto() + request.name = "my-field" + request.number = 123 + + transcoded, body, query_params = transcode_request(http_options, request) + + assert transcoded["method"] == "get" + assert transcoded["uri"] == "/v1/test/my-field" + assert body is None + # 'number' should be in query parameters + assert "number" in query_params + assert query_params["number"] == 123 + + +def test_transcode_with_nested_field(): + http_options = [ + { + "method": "get", + "uri": "/v1/test/{options.deprecated}/{name}", + } + ] + + request = descriptor_pb2.FieldDescriptorProto() + request.name = "my-field" + request.options.deprecated = True + request.number = 123 + + transcoded, body, query_params = transcode_request(http_options, request) + + assert transcoded["method"] == "get" + assert transcoded["uri"] == "/v1/test/True/my-field" + assert body is None + assert "number" in query_params + assert query_params["number"] == 123 + + +def test_transcode_with_body(): + http_options = [ + { + "method": "post", + "uri": "/v1/test/{name}", + "body": "options", + } + ] + + request = descriptor_pb2.FieldDescriptorProto() + request.name = "my-field" + request.options.deprecated = True + request.number = 123 + + transcoded, body, query_params = transcode_request(http_options, request) + + assert transcoded["method"] == "post" + assert transcoded["uri"] == "/v1/test/my-field" + assert body is not None + body_data = json.loads(body) + assert body_data["deprecated"] is True + # Query parameters should not contain 'options' (the body) + assert "number" in query_params + assert query_params["number"] == 123 + assert "options" not in query_params + + +def test_transcode_with_required_fields_default_values(): + http_options = [ + { + "method": "get", + "uri": "/v1/test/{name}", + } + ] + + request = descriptor_pb2.FieldDescriptorProto() + request.name = "my-field" + + required_defaults = {"requiredQueryParam": "default-val"} + + transcoded, body, query_params = transcode_request( + http_options, + request, + required_fields_default_values=required_defaults, + ) + + assert query_params["requiredQueryParam"] == "default-val" + + +def test_transcode_with_numeric_enums(): + http_options = [ + { + "method": "get", + "uri": "/v1/test/{name}", + } + ] + + request = descriptor_pb2.FieldDescriptorProto() + request.name = "my-field" + request.type = descriptor_pb2.FieldDescriptorProto.TYPE_STRING + + # Without numeric enums + _, _, query_params = transcode_request(http_options, request, rest_numeric_enums=False) + assert query_params["type"] == "TYPE_STRING" + + # With numeric enums + _, _, query_params = transcode_request(http_options, request, rest_numeric_enums=True) + # Type number for TYPE_STRING is 9 + assert query_params["type"] == 9 + assert query_params["$alt"] == "json;enum-encoding=int"