diff --git a/packages/gapic-generator/gapic/generator/generator.py b/packages/gapic-generator/gapic/generator/generator.py index 9fe56aa9de1d..c1932abbe426 100644 --- a/packages/gapic-generator/gapic/generator/generator.py +++ b/packages/gapic-generator/gapic/generator/generator.py @@ -120,7 +120,7 @@ def get_response(self, api_schema: api.API, opts: Options) -> CodeGeneratorRespo for template_name in client_templates: # Quick check: Skip "private" templates. filename = template_name.split("/")[-1] - if filename.startswith("_") and filename != "__init__.py.j2": + if filename.startswith("_") and filename not in ("__init__.py.j2", "_compat.py.j2"): continue # Append to the output files dictionary. diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/_compat.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/_compat.py.j2 new file mode 100644 index 000000000000..1dd644e778ff --- /dev/null +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/_compat.py.j2 @@ -0,0 +1,36 @@ +# {% include '_license.j2' %} + +import uuid + +try: + from google.api_core.gapic_v1.requests import setup_request_id +except ImportError: + # TODO: Remove this fallback when google-api-core >= 2.18.0 is the minimum required version. + def setup_request_id(request, field_name: str, is_proto3_optional: bool): + """Populate a UUID4 field in the request if it is not already set. + + Args: + request (Union[google.protobuf.message.Message, dict]): The request object. + field_name (str): The name of the field to populate. + is_proto3_optional (bool): Whether the field is proto3 optional. + """ + if isinstance(request, dict): + if is_proto3_optional: + if field_name not in request: + request[field_name] = str(uuid.uuid4()) + elif not request.get(field_name): + request[field_name] = str(uuid.uuid4()) + return + + if is_proto3_optional: + try: + # Pure protobuf messages + if not request.HasField(field_name): + setattr(request, field_name, str(uuid.uuid4())) + except (AttributeError, ValueError): + # Proto-plus messages or other objects + if field_name not in request: + setattr(request, field_name, str(uuid.uuid4())) + else: + if not getattr(request, field_name): + setattr(request, field_name, str(uuid.uuid4())) diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2 index 755e4530e7ba..5f1b236f1780 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2 @@ -28,11 +28,7 @@ {% if method_settings is not none %} {% for auto_populated_field in method_settings.auto_populated_fields %} {% set is_proto3_optional = method.input.fields[auto_populated_field].proto3_optional %} - {% if is_async %} - self._client._setup_request_id(request, '{{ auto_populated_field }}', {{ is_proto3_optional }}) - {% else %} - self._setup_request_id(request, '{{ auto_populated_field }}', {{ is_proto3_optional }}) - {% endif %} + setup_request_id(request, '{{ auto_populated_field }}', {{ is_proto3_optional }}) {% endfor %} {% endif %}{# if method_settings is not none #} {% endwith %}{# method_settings #} diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 index 5fe416902b86..759a4a052c54 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 @@ -4,13 +4,18 @@ {% import "%namespace/%name_%version/%sub/services/%service/_client_macros.j2" as macros %} {% import "%namespace/%name_%version/%sub/services/%service/_shared_macros.j2" as shared_macros %} +{% set has_auto_populated_fields = namespace(value=false) %} +{% for method in service.methods.values() %} + {% set method_settings = api.all_method_settings.get(method.meta.address.proto) %} + {% if method_settings and method_settings.auto_populated_fields %} + {% set has_auto_populated_fields.value = true %} + {% endif %} +{% endfor %} + import logging as std_logging from collections import OrderedDict import re from typing import Dict, Callable, Mapping, MutableMapping, MutableSequence, Optional, {% if service.any_server_streaming %}AsyncIterable, Awaitable, {% endif %}{% if service.any_client_streaming %}AsyncIterator, {% endif %}Sequence, Tuple, Type, Union -{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %} -import uuid -{% endif %} {% if service.any_deprecated %} import warnings {% endif %} @@ -21,6 +26,9 @@ from {{package_path}} import gapic_version as package_version from google.api_core.client_options import ClientOptions from google.api_core import exceptions as core_exceptions from google.api_core import gapic_v1 +{% if has_auto_populated_fields.value %} +from {{package_path}}._compat import setup_request_id +{% endif %} from google.api_core import retry_async as retries from google.auth import credentials as ga_credentials # type: ignore from google.oauth2 import service_account # type: ignore diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 index e2e3edb24967..cfeb327a486d 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 @@ -6,6 +6,14 @@ {% import "%namespace/%name_%version/%sub/services/%service/_client_macros.j2" as macros %} {% import "%namespace/%name_%version/%sub/services/%service/_shared_macros.j2" as shared_macros %} +{% set has_auto_populated_fields = namespace(value=false) %} +{% for method in service.methods.values() %} + {% set method_settings = api.all_method_settings.get(method.meta.address.proto) %} + {% if method_settings and method_settings.auto_populated_fields %} + {% set has_auto_populated_fields.value = true %} + {% endif %} +{% endfor %} + from collections import OrderedDict {% if service.any_extended_operations_methods %} import functools @@ -16,9 +24,6 @@ import logging as std_logging import os import re from typing import Dict, Callable, Mapping, MutableMapping, MutableSequence, Optional, {% if service.any_server_streaming %}Iterable, {% endif %}{% if service.any_client_streaming %}Iterator, {% endif %}Sequence, Tuple, Type, Union, cast -{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %} -import uuid -{% endif %} import warnings {% set package_path = api.naming.module_namespace|join('.') + "." + api.naming.versioned_module_name %} @@ -30,6 +35,9 @@ from google.api_core import exceptions as core_exceptions from google.api_core import extended_operation {% endif %} from google.api_core import gapic_v1 +{% if has_auto_populated_fields.value %} +from {{package_path}}._compat import setup_request_id +{% endif %} from google.api_core import retry as retries from google.auth import credentials as ga_credentials # type: ignore from google.auth.transport import mtls # type: ignore @@ -453,37 +461,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta): # NOTE (b/349488459): universe validation is disabled until further notice. return True - {% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %} - @staticmethod - def _setup_request_id(request, field_name: str, is_proto3_optional: bool): - """Populate a UUID4 field in the request if it is not already set. - - Args: - request (Union[google.protobuf.message.Message, dict]): The request object. - field_name (str): The name of the field to populate. - is_proto3_optional (bool): Whether the field is proto3 optional. - """ - if isinstance(request, dict): - if is_proto3_optional: - if field_name not in request: - request[field_name] = str(uuid.uuid4()) - elif not request.get(field_name): - request[field_name] = str(uuid.uuid4()) - return - if is_proto3_optional: - try: - # Pure protobuf messages - if not request.HasField(field_name): - setattr(request, field_name, str(uuid.uuid4())) - except (AttributeError, ValueError): - # Proto-plus messages or other objects - if field_name not in request: - setattr(request, field_name, str(uuid.uuid4())) - else: - if not getattr(request, field_name): - setattr(request, field_name, str(uuid.uuid4())) - {% endif %} def _add_cred_info_for_auth_errors( self, diff --git a/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 b/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 index c68b390b7c84..aa7c48173f5c 100644 --- a/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 +++ b/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 @@ -6,9 +6,17 @@ {% import "tests/unit/gapic/%name_%version/%sub/test_macros.j2" as test_macros %} {% import "%namespace/%name_%version/%sub/services/%service/_shared_macros.j2" as shared_macros %} +{% set has_auto_populated_fields = namespace(value=false) %} +{% for method in service.methods.values() %} + {% set method_settings = api.all_method_settings.get(method.meta.address.proto) %} + {% if method_settings and method_settings.auto_populated_fields %} + {% set has_auto_populated_fields.value = true %} + {% endif %} +{% endfor %} + import os import asyncio -{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %} +{% if has_auto_populated_fields.value %} import re {% endif %} from unittest import mock @@ -107,7 +115,7 @@ CRED_INFO_JSON = { "principal": "service-account@example.com", } CRED_INFO_STRING = json.dumps(CRED_INFO_JSON) -{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %} +{% if has_auto_populated_fields.value %} _UUID4_RE = re.compile(r"{{ uuid4_re }}") {% endif %} @@ -434,84 +442,6 @@ def test__add_cred_info_for_auth_errors_no_get_cred_info(error_code): client._add_cred_info_for_auth_errors(error) assert error.details == [] -{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %} -def test__setup_request_id(): - class MockRequest: - def __init__(self, **kwargs): - for k, v in kwargs.items(): - setattr(self, k, v) - def __contains__(self, key): - return hasattr(self, key) - - class MockProtoRequest: - def __init__(self, **kwargs): - for k, v in kwargs.items(): - setattr(self, k, v) - def HasField(self, key): - return hasattr(self, key) - - # Test with proto3 optional field not in request - request = MockRequest() - {{ service.client_name }}._setup_request_id(request, "request_id", True) - assert re.match(r"{{ test_macros.get_uuid4_re() }}", request.request_id) - - # Test with proto3 optional field already in request - request = MockRequest(request_id="already_set") - {{ service.client_name }}._setup_request_id(request, "request_id", True) - assert request.request_id == "already_set" - - # Test with non-proto3 optional field empty - request = MockRequest(request_id="") - {{ service.client_name }}._setup_request_id(request, "request_id", False) - assert re.match(r"{{ test_macros.get_uuid4_re() }}", request.request_id) - - # Test with non-proto3 optional field already set - request = MockRequest(request_id="already_set") - {{ service.client_name }}._setup_request_id(request, "request_id", False) - assert request.request_id == "already_set" - - # Test with proto3 optional field not in request (MockProtoRequest) - request = MockProtoRequest() - {{ service.client_name }}._setup_request_id(request, "request_id", True) - assert re.match(r"{{ test_macros.get_uuid4_re() }}", request.request_id) - - # Test with proto3 optional field already in request (MockProtoRequest) - request = MockProtoRequest(request_id="already_set") - {{ service.client_name }}._setup_request_id(request, "request_id", True) - assert request.request_id == "already_set" - - # Test with ValueError - class MockValueErrorRequest: - def HasField(self, key): - raise ValueError("Mismatched field") - def __contains__(self, key): - return hasattr(self, key) - - request = MockValueErrorRequest() - {{ service.client_name }}._setup_request_id(request, "request_id", True) - assert re.match(r"{{ test_macros.get_uuid4_re() }}", request.request_id) - - # Test with dict and proto3 optional field not in request - request = {} - {{ service.client_name }}._setup_request_id(request, "request_id", True) - assert re.match(r"{{ test_macros.get_uuid4_re() }}", request["request_id"]) - - # Test with dict and proto3 optional field already in request - request = {"request_id": "already_set"} - {{ service.client_name }}._setup_request_id(request, "request_id", True) - assert request["request_id"] == "already_set" - - # Test with dict and non-proto3 optional field empty - request = {"request_id": ""} - {{ service.client_name }}._setup_request_id(request, "request_id", False) - assert re.match(r"{{ test_macros.get_uuid4_re() }}", request["request_id"]) - - # Test with dict and non-proto3 optional field already set - request = {"request_id": "already_set"} - {{ service.client_name }}._setup_request_id(request, "request_id", False) - assert request["request_id"] == "already_set" - -{% endif %} @pytest.mark.parametrize("client_class,transport_name", [ {% if 'grpc' in opts.transport %} ({{ service.client_name }}, "grpc"), diff --git a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/google/cloud/storagebatchoperations_v1/services/storage_batch_operations/client.py b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/google/cloud/storagebatchoperations_v1/services/storage_batch_operations/client.py index 5f79cf8e016a..27c7188f4a3e 100755 --- a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/google/cloud/storagebatchoperations_v1/services/storage_batch_operations/client.py +++ b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/google/cloud/storagebatchoperations_v1/services/storage_batch_operations/client.py @@ -28,6 +28,7 @@ from google.api_core import client_options as client_options_lib from google.api_core import exceptions as core_exceptions from google.api_core import gapic_v1 +from google.api_core.gapic_v1.requests import setup_request_id from google.api_core import retry as retries from google.auth import credentials as ga_credentials # type: ignore from google.auth.transport import mtls # type: ignore @@ -480,26 +481,7 @@ def _setup_request_id(request, field_name: str, is_proto3_optional: bool): field_name (str): The name of the field to populate. is_proto3_optional (bool): Whether the field is proto3 optional. """ - if isinstance(request, dict): - if is_proto3_optional: - if field_name not in request: - request[field_name] = str(uuid.uuid4()) - elif not request.get(field_name): - request[field_name] = str(uuid.uuid4()) - return - - if is_proto3_optional: - try: - # Pure protobuf messages - if not request.HasField(field_name): - setattr(request, field_name, str(uuid.uuid4())) - except (AttributeError, ValueError): - # Proto-plus messages or other objects - if field_name not in request: - setattr(request, field_name, str(uuid.uuid4())) - else: - if not getattr(request, field_name): - setattr(request, field_name, str(uuid.uuid4())) + setup_request_id(request, field_name, is_proto3_optional) def _add_cred_info_for_auth_errors( self, diff --git a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/tests/unit/gapic/storagebatchoperations_v1/test_storage_batch_operations.py b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/tests/unit/gapic/storagebatchoperations_v1/test_storage_batch_operations.py index 5c53e97f8d12..3697bd9a4839 100755 --- a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/tests/unit/gapic/storagebatchoperations_v1/test_storage_batch_operations.py +++ b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/tests/unit/gapic/storagebatchoperations_v1/test_storage_batch_operations.py @@ -351,82 +351,6 @@ def test__add_cred_info_for_auth_errors_no_get_cred_info(error_code): client._add_cred_info_for_auth_errors(error) assert error.details == [] -def test__setup_request_id(): - class MockRequest: - def __init__(self, **kwargs): - for k, v in kwargs.items(): - setattr(self, k, v) - def __contains__(self, key): - return hasattr(self, key) - - class MockProtoRequest: - def __init__(self, **kwargs): - for k, v in kwargs.items(): - setattr(self, k, v) - def HasField(self, key): - return hasattr(self, key) - - # Test with proto3 optional field not in request - request = MockRequest() - StorageBatchOperationsClient._setup_request_id(request, "request_id", True) - assert re.match(r"[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}", request.request_id) - - # Test with proto3 optional field already in request - request = MockRequest(request_id="already_set") - StorageBatchOperationsClient._setup_request_id(request, "request_id", True) - assert request.request_id == "already_set" - - # Test with non-proto3 optional field empty - request = MockRequest(request_id="") - StorageBatchOperationsClient._setup_request_id(request, "request_id", False) - assert re.match(r"[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}", request.request_id) - - # Test with non-proto3 optional field already set - request = MockRequest(request_id="already_set") - StorageBatchOperationsClient._setup_request_id(request, "request_id", False) - assert request.request_id == "already_set" - - # Test with proto3 optional field not in request (MockProtoRequest) - request = MockProtoRequest() - StorageBatchOperationsClient._setup_request_id(request, "request_id", True) - assert re.match(r"[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}", request.request_id) - - # Test with proto3 optional field already in request (MockProtoRequest) - request = MockProtoRequest(request_id="already_set") - StorageBatchOperationsClient._setup_request_id(request, "request_id", True) - assert request.request_id == "already_set" - - # Test with ValueError - class MockValueErrorRequest: - def HasField(self, key): - raise ValueError("Mismatched field") - def __contains__(self, key): - return hasattr(self, key) - - request = MockValueErrorRequest() - StorageBatchOperationsClient._setup_request_id(request, "request_id", True) - assert re.match(r"[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}", request.request_id) - - # Test with dict and proto3 optional field not in request - request = {} - StorageBatchOperationsClient._setup_request_id(request, "request_id", True) - assert re.match(r"[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}", request["request_id"]) - - # Test with dict and proto3 optional field already in request - request = {"request_id": "already_set"} - StorageBatchOperationsClient._setup_request_id(request, "request_id", True) - assert request["request_id"] == "already_set" - - # Test with dict and non-proto3 optional field empty - request = {"request_id": ""} - StorageBatchOperationsClient._setup_request_id(request, "request_id", False) - assert re.match(r"[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}", request["request_id"]) - - # Test with dict and non-proto3 optional field already set - request = {"request_id": "already_set"} - StorageBatchOperationsClient._setup_request_id(request, "request_id", False) - assert request["request_id"] == "already_set" - @pytest.mark.parametrize("client_class,transport_name", [ (StorageBatchOperationsClient, "grpc"), (StorageBatchOperationsAsyncClient, "grpc_asyncio"),