1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
|
# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
import copy
from botocore.compat import OrderedDict
from botocore.endpoint import DEFAULT_TIMEOUT, MAX_POOL_CONNECTIONS
from botocore.exceptions import (
InvalidMaxRetryAttemptsError,
InvalidRetryConfigurationError,
InvalidRetryModeError,
InvalidS3AddressingStyleError,
)
class Config:
"""Advanced configuration for Botocore clients.
:type region_name: str
:param region_name: The region to use in instantiating the client
:type signature_version: str
:param signature_version: The signature version when signing requests.
:type user_agent: str
:param user_agent: The value to use in the User-Agent header.
:type user_agent_extra: str
:param user_agent_extra: The value to append to the current User-Agent
header value.
:type user_agent_appid: str
:param user_agent_appid: A value that gets included in the User-Agent
string in the format "app/<user_agent_appid>". Allowed characters are
ASCII alphanumerics and ``!$%&'*+-.^_`|~``. All other characters will
be replaced by a ``-``.
:type connect_timeout: float or int
:param connect_timeout: The time in seconds till a timeout exception is
thrown when attempting to make a connection. The default is 60
seconds.
:type read_timeout: float or int
:param read_timeout: The time in seconds till a timeout exception is
thrown when attempting to read from a connection. The default is
60 seconds.
:type parameter_validation: bool
:param parameter_validation: Whether parameter validation should occur
when serializing requests. The default is True. You can disable
parameter validation for performance reasons. Otherwise, it's
recommended to leave parameter validation enabled.
:type max_pool_connections: int
:param max_pool_connections: The maximum number of connections to
keep in a connection pool. If this value is not set, the default
value of 10 is used.
:type proxies: dict
:param proxies: A dictionary of proxy servers to use by protocol or
endpoint, e.g.:
``{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}``.
The proxies are used on each request.
:type proxies_config: dict
:param proxies_config: A dictionary of additional proxy configurations.
Valid keys are:
* ``proxy_ca_bundle`` -- The path to a custom certificate bundle to use
when establishing SSL/TLS connections with proxy.
* ``proxy_client_cert`` -- The path to a certificate for proxy
TLS client authentication.
When a string is provided it is treated as a path to a proxy client
certificate. When a two element tuple is provided, it will be
interpreted as the path to the client certificate, and the path
to the certificate key.
* ``proxy_use_forwarding_for_https`` -- For HTTPS proxies,
forward your requests to HTTPS destinations with an absolute
URI. We strongly recommend you only use this option with
trusted or corporate proxies. Value must be boolean.
:type s3: dict
:param s3: A dictionary of S3 specific configurations.
Valid keys are:
* ``use_accelerate_endpoint`` -- Refers to whether to use the S3
Accelerate endpoint. The value must be a boolean. If True, the
client will use the S3 Accelerate endpoint. If the S3 Accelerate
endpoint is being used then the addressing style will always
be virtual.
* ``payload_signing_enabled`` -- Refers to whether or not to SHA256
sign sigv4 payloads. By default, this is disabled for streaming
uploads (UploadPart and PutObject).
* ``addressing_style`` -- Refers to the style in which to address
s3 endpoints. Values must be a string that equals one of:
* ``auto`` -- Addressing style is chosen for user. Depending
on the configuration of client, the endpoint may be addressed in
the virtual or the path style. Note that this is the default
behavior if no style is specified.
* ``virtual`` -- Addressing style is always virtual. The name of the
bucket must be DNS compatible or an exception will be thrown.
Endpoints will be addressed as such: ``amzn-s3-demo-bucket.s3.amazonaws.com``
* ``path`` -- Addressing style is always by path. Endpoints will be
addressed as such: ``s3.amazonaws.com/amzn-s3-demo-bucket``
* ``us_east_1_regional_endpoint`` -- Refers to what S3 endpoint to use
when the region is configured to be us-east-1. Values must be a
string that equals:
* ``regional`` -- Use the us-east-1.amazonaws.com endpoint if the
client is configured to use the us-east-1 region.
* ``legacy`` -- Use the s3.amazonaws.com endpoint if the client is
configured to use the us-east-1 region. This is the default if
the configuration option is not specified.
:type retries: dict
:param retries: A dictionary for configuration related to retry behavior.
Valid keys are:
* ``total_max_attempts`` -- An integer representing the maximum number of
total attempts that will be made on a single request. This includes
the initial request, so a value of 1 indicates that no requests
will be retried. If ``total_max_attempts`` and ``max_attempts``
are both provided, ``total_max_attempts`` takes precedence.
``total_max_attempts`` is preferred over ``max_attempts`` because
it maps to the ``AWS_MAX_ATTEMPTS`` environment variable and
the ``max_attempts`` config file value.
* ``max_attempts`` -- An integer representing the maximum number of
retry attempts that will be made on a single request. For
example, setting this value to 2 will result in the request
being retried at most two times after the initial request. Setting
this value to 0 will result in no retries ever being attempted after
the initial request. If not provided, the number of retries will
default to the value specified in the service model, which is
typically four retries.
* ``mode`` -- A string representing the type of retry mode botocore
should use. Valid values are:
* ``legacy`` - The pre-existing retry behavior.
* ``standard`` - The standardized set of retry rules. This will also
default to 3 max attempts unless overridden.
* ``adaptive`` - Retries with additional client side throttling.
:type client_cert: str, (str, str)
:param client_cert: The path to a certificate for TLS client authentication.
When a string is provided it is treated as a path to a client
certificate to be used when creating a TLS connection.
If a client key is to be provided alongside the client certificate the
client_cert should be set to a tuple of length two where the first
element is the path to the client certificate and the second element is
the path to the certificate key.
:type inject_host_prefix: bool
:param inject_host_prefix: Whether host prefix injection should occur.
Defaults to None.
The default of None is equivalent to setting to True, which enables
the injection of operation parameters into the prefix of the hostname.
Setting this to False disables the injection of operation parameters
into the prefix of the hostname. Setting this to False is useful for
clients providing custom endpoints that should not have their host
prefix modified.
:type use_dualstack_endpoint: bool
:param use_dualstack_endpoint: Setting to True enables dualstack
endpoint resolution.
Defaults to None.
:type use_fips_endpoint: bool
:param use_fips_endpoint: Setting to True enables fips
endpoint resolution.
Defaults to None.
:type ignore_configured_endpoint_urls: bool
:param ignore_configured_endpoint_urls: Setting to True disables use
of endpoint URLs provided via environment variables and
the shared configuration file.
Defaults to None.
:type tcp_keepalive: bool
:param tcp_keepalive: Enables the TCP Keep-Alive socket option used when
creating new connections if set to True.
Defaults to False.
:type request_min_compression_size_bytes: int
:param request_min_compression_size_bytes: The minimum size in bytes that a
request body should be to trigger compression. All requests with
streaming input that don't contain the ``requiresLength`` trait will be
compressed regardless of this setting.
Defaults to None.
:type disable_request_compression: bool
:param disable_request_compression: Disables request body compression if
set to True.
Defaults to None.
:type sigv4a_signing_region_set: string
:param sigv4a_signing_region_set: A set of AWS regions to apply the signature for
when using SigV4a for signing. Set to ``*`` to represent all regions.
Defaults to None.
:type client_context_params: dict
:param client_context_params: A dictionary of parameters specific to
individual services. If available, valid parameters can be found in
the ``Client Context Parameters`` section of the service client's
documentation. Invalid parameters or ones that are not used by the
specified service will be ignored.
Defaults to None.
:type request_checksum_calculation: str
:param request_checksum_calculation: Determines when a checksum will be
calculated for request payloads. Valid values are:
* ``when_supported`` -- When set, a checksum will be calculated for
all request payloads of operations modeled with the ``httpChecksum``
trait where ``requestChecksumRequired`` is ``true`` or a
``requestAlgorithmMember`` is modeled.
* ``when_required`` -- When set, a checksum will only be calculated
for request payloads of operations modeled with the ``httpChecksum``
trait where ``requestChecksumRequired`` is ``true`` or where a
``requestAlgorithmMember`` is modeled and supplied.
Defaults to None.
:type response_checksum_validation: str
:param response_checksum_validation: Determines when checksum validation
will be performed on response payloads. Valid values are:
* ``when_supported`` -- When set, checksum validation is performed on
all response payloads of operations modeled with the ``httpChecksum``
trait where ``responseAlgorithms`` is modeled, except when no modeled
checksum algorithms are supported.
* ``when_required`` -- When set, checksum validation is not performed
on response payloads of operations unless the checksum algorithm is
supported and the ``requestValidationModeMember`` member is set to ``ENABLED``.
Defaults to None.
:type account_id_endpoint_mode: str
:param account_id_endpoint_mode: The value used to determine the client's
behavior for account ID based endpoint routing. Valid values are:
* ``preferred`` - The endpoint should include account ID if available.
* ``disabled`` - A resolved endpoint does not include account ID.
* ``required`` - The endpoint must include account ID. If the account ID
isn't available, an exception will be raised.
If a value is not provided, the client will default to ``preferred``.
Defaults to None.
"""
OPTION_DEFAULTS = OrderedDict(
[
('region_name', None),
('signature_version', None),
('user_agent', None),
('user_agent_extra', None),
('user_agent_appid', None),
('connect_timeout', DEFAULT_TIMEOUT),
('read_timeout', DEFAULT_TIMEOUT),
('parameter_validation', True),
('max_pool_connections', MAX_POOL_CONNECTIONS),
('proxies', None),
('proxies_config', None),
('s3', None),
('retries', None),
('client_cert', None),
('inject_host_prefix', None),
('endpoint_discovery_enabled', None),
('use_dualstack_endpoint', None),
('use_fips_endpoint', None),
('ignore_configured_endpoint_urls', None),
('defaults_mode', None),
('tcp_keepalive', None),
('request_min_compression_size_bytes', None),
('disable_request_compression', None),
('client_context_params', None),
('sigv4a_signing_region_set', None),
('request_checksum_calculation', None),
('response_checksum_validation', None),
('account_id_endpoint_mode', None),
]
)
NON_LEGACY_OPTION_DEFAULTS = {
'connect_timeout': None,
}
# The original default value of the inject_host_prefix parameter was True.
# This prevented the ability to override the value from other locations in
# the parameter provider chain, like env vars or the shared configuration
# file. TO accomplish this, we need to disambiguate when the value was set
# by the user or not. This overrides the parameter with a property so the
# default value of inject_host_prefix is still True if it is not set by the
# user.
@property
def inject_host_prefix(self):
if self._inject_host_prefix == "UNSET":
return True
return self._inject_host_prefix
# Override the setter for the case where the user does supply a value;
# _inject_host_prefix will no longer be "UNSET".
@inject_host_prefix.setter
def inject_host_prefix(self, value):
self._inject_host_prefix = value
def __init__(self, *args, **kwargs):
self._user_provided_options = self._record_user_provided_options(
args, kwargs
)
# By default, we use a value that indicates the user did not
# set it. This value MUST persist on the Config object to be used
# elsewhere.
self._inject_host_prefix = 'UNSET'
# Merge the user_provided options onto the default options
config_vars = copy.copy(self.OPTION_DEFAULTS)
defaults_mode = self._user_provided_options.get(
'defaults_mode', 'legacy'
)
if defaults_mode != 'legacy':
config_vars.update(self.NON_LEGACY_OPTION_DEFAULTS)
config_vars.update(self._user_provided_options)
# Set the attributes based on the config_vars
for key, value in config_vars.items():
# Default values for the Config object are set here. We don't want
# to use `setattr` in the case where the user already supplied a
# value.
if (
key == 'inject_host_prefix'
and 'inject_host_prefix'
not in self._user_provided_options.keys()
):
continue
setattr(self, key, value)
# Validate the s3 options
self._validate_s3_configuration(self.s3)
self._validate_retry_configuration(self.retries)
def _record_user_provided_options(self, args, kwargs):
option_order = list(self.OPTION_DEFAULTS)
user_provided_options = {}
# Iterate through the kwargs passed through to the constructor and
# map valid keys to the dictionary
for key, value in kwargs.items():
if key in self.OPTION_DEFAULTS:
user_provided_options[key] = value
# The key must exist in the available options
else:
raise TypeError(f"Got unexpected keyword argument '{key}'")
# The number of args should not be longer than the allowed
# options
if len(args) > len(option_order):
raise TypeError(
f"Takes at most {len(option_order)} arguments ({len(args)} given)"
)
# Iterate through the args passed through to the constructor and map
# them to appropriate keys.
for i, arg in enumerate(args):
# If a kwarg was specified for the arg, then error out
if option_order[i] in user_provided_options:
raise TypeError(
f"Got multiple values for keyword argument '{option_order[i]}'"
)
user_provided_options[option_order[i]] = arg
return user_provided_options
def _validate_s3_configuration(self, s3):
if s3 is not None:
addressing_style = s3.get('addressing_style')
if addressing_style not in ['virtual', 'auto', 'path', None]:
raise InvalidS3AddressingStyleError(
s3_addressing_style=addressing_style
)
def _validate_retry_configuration(self, retries):
valid_options = ('max_attempts', 'mode', 'total_max_attempts')
valid_modes = ('legacy', 'standard', 'adaptive')
if retries is not None:
for key, value in retries.items():
if key not in valid_options:
raise InvalidRetryConfigurationError(
retry_config_option=key,
valid_options=valid_options,
)
if key == 'max_attempts' and value < 0:
raise InvalidMaxRetryAttemptsError(
provided_max_attempts=value,
min_value=0,
)
if key == 'total_max_attempts' and value < 1:
raise InvalidMaxRetryAttemptsError(
provided_max_attempts=value,
min_value=1,
)
if key == 'mode' and value not in valid_modes:
raise InvalidRetryModeError(
provided_retry_mode=value,
valid_modes=valid_modes,
)
def merge(self, other_config):
"""Merges the config object with another config object
This will merge in all non-default values from the provided config
and return a new config object
:type other_config: botocore.config.Config
:param other config: Another config object to merge with. The values
in the provided config object will take precedence in the merging
:returns: A config object built from the merged values of both
config objects.
"""
# Make a copy of the current attributes in the config object.
config_options = copy.copy(self._user_provided_options)
# Merge in the user provided options from the other config
config_options.update(other_config._user_provided_options)
# Return a new config object with the merged properties.
return Config(**config_options)
|