googleapiclient.discovery

Package googleapiclient :: Module discovery |
[hide private]
[frames] | no frames]

# Source Code for Module googleapiclient.discovery

       1  # Copyright 2014 Google Inc. All Rights Reserved.
       2  #
       3  # Licensed under the Apache License, Version 2.0 (the "License");
       4  # you may not use this file except in compliance with the License.
       5  # You may obtain a copy of the License at
       6  #
       7  #      http://www.apache.org/licenses/LICENSE-2.0
       8  #
       9  # Unless required by applicable law or agreed to in writing, software
      10  # distributed under the License is distributed on an "AS IS" BASIS,
      11  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12  # See the License for the specific language governing permissions and
      13  # limitations under the License.
      14
      15  """Client for discovery based APIs.
      16
      17  A client library for Google's discovery based APIs.
      18  """
      19  from __future__ import absolute_import
      20  import six
      21  from six.moves import zip
      22
      23  __author__ = 'jcgregorio@google.com (Joe Gregorio)'
      24  __all__ = [
      25      'build',
      26      'build_from_document',
      27      'fix_method_name',
      28      'key2param',
      29      ]
      30
      31  from six import BytesIO
      32  from six.moves import http_client
      33  from six.moves.urllib.parse import urlencode, urlparse, urljoin, \
      34    urlunparse, parse_qsl
      35
      36  # Standard library imports
      37  import copy
      38  try:
      39    from email.generator import BytesGenerator
      40  except ImportError:
      41    from email.generator import Generator as BytesGenerator
      42  from email.mime.multipart import MIMEMultipart
      43  from email.mime.nonmultipart import MIMENonMultipart
      44  import json
      45  import keyword
      46  import logging
      47  import mimetypes
      48  import os
      49  import re
      50
      51  # Third-party imports
      52  import httplib2
      53  import uritemplate
      54
      55  # Local imports
      56  from googleapiclient import _auth
      57  from googleapiclient import mimeparse
      58  from googleapiclient.errors import HttpError
      59  from googleapiclient.errors import InvalidJsonError
      60  from googleapiclient.errors import MediaUploadSizeError
      61  from googleapiclient.errors import UnacceptableMimeTypeError
      62  from googleapiclient.errors import UnknownApiNameOrVersion
      63  from googleapiclient.errors import UnknownFileType
      64  from googleapiclient.http import build_http
      65  from googleapiclient.http import BatchHttpRequest
      66  from googleapiclient.http import HttpMock
      67  from googleapiclient.http import HttpMockSequence
      68  from googleapiclient.http import HttpRequest
      69  from googleapiclient.http import MediaFileUpload
      70  from googleapiclient.http import MediaUpload
      71  from googleapiclient.model import JsonModel
      72  from googleapiclient.model import MediaModel
      73  from googleapiclient.model import RawModel
      74  from googleapiclient.schema import Schemas
      75
      76  from googleapiclient._helpers import _add_query_parameter
      77  from googleapiclient._helpers import positional
      78
      79
      80  # The client library requires a version of httplib2 that supports RETRIES.
      81  httplib2.RETRIES = 1
      82
      83  logger = logging.getLogger(__name__)
      84
      85  URITEMPLATE = re.compile('{[^}]*}')
      86  VARNAME = re.compile('[a-zA-Z0-9_-]+')
      87  DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/'
      88                   '{api}/{apiVersion}/rest')
      89  V1_DISCOVERY_URI = DISCOVERY_URI
      90  V2_DISCOVERY_URI = ('https://{api}.googleapis.com/$discovery/rest?'
      91                      'version={apiVersion}')
      92  DEFAULT_METHOD_DOC = 'A description of how to use this function'
      93  HTTP_PAYLOAD_METHODS = frozenset(['PUT', 'POST', 'PATCH'])
      94
      95  _MEDIA_SIZE_BIT_SHIFTS = {'KB': 10, 'MB': 20, 'GB': 30, 'TB': 40}
      96  BODY_PARAMETER_DEFAULT_VALUE = {
      97      'description': 'The request body.',
      98      'type': 'object',
      99      'required': True,
     100  }
     101  MEDIA_BODY_PARAMETER_DEFAULT_VALUE = {
     102      'description': ('The filename of the media request body, or an instance '
     103                      'of a MediaUpload object.'),
     104      'type': 'string',
     105      'required': False,
     106  }
     107  MEDIA_MIME_TYPE_PARAMETER_DEFAULT_VALUE = {
     108      'description': ('The MIME type of the media request body, or an instance '
     109                      'of a MediaUpload object.'),
     110      'type': 'string',
     111      'required': False,
     112  }
     113  _PAGE_TOKEN_NAMES = ('pageToken', 'nextPageToken')
     114
     115  # Parameters accepted by the stack, but not visible via discovery.
     116  # TODO(dhermes): Remove 'userip' in 'v2'.
     117  STACK_QUERY_PARAMETERS = frozenset(['trace', 'pp', 'userip', 'strict'])
     118  STACK_QUERY_PARAMETER_DEFAULT_VALUE = {'type': 'string', 'location': 'query'}
     119
     120  # Library-specific reserved words beyond Python keywords.
     121  RESERVED_WORDS = frozenset(['body'])
     122
     123  # patch _write_lines to avoid munging '\r' into '\n'
     124  # ( https://bugs.python.org/issue18886 https://bugs.python.org/issue19003 )
     125 -class _BytesGenerator(BytesGenerator):
     126    _write_lines = BytesGenerator.write
     127
     128 -def fix_method_name(name):
     129    """Fix method names to avoid '$' characters and reserved word conflicts.
     130
     131    Args:
     132      name: string, method name.
     133
     134    Returns:
     135      The name with '_' appended if the name is a reserved word and '$'
     136      replaced with '_'.
     137    """
     138    name = name.replace('$', '_')
     139    if keyword.iskeyword(name) or name in RESERVED_WORDS:
     140      return name + '_'
     141    else:
     142      return name
     143
     144
     145 -def key2param(key):
     146    """Converts key names into parameter names.
     147
     148    For example, converting "max-results" -> "max_results"
     149
     150    Args:
     151      key: string, the method key name.
     152
     153    Returns:
     154      A safe method name based on the key name.
     155    """
     156    result = []
     157    key = list(key)
     158    if not key[0].isalpha():
     159      result.append('x')
     160    for c in key:
     161      if c.isalnum():
     162        result.append(c)
     163      else:
     164        result.append('_')
     165
     166    return ''.join(result)
     167
     168
     169  @positional(2)
     170 -def build(serviceName,
     171            version,
     172            http=None,
     173            discoveryServiceUrl=DISCOVERY_URI,
     174            developerKey=None,
     175            model=None,
     176            requestBuilder=HttpRequest,
     177            credentials=None,
     178            cache_discovery=True,
     179            cache=None):
     180    """Construct a Resource for interacting with an API.
     181
     182    Construct a Resource object for interacting with an API. The serviceName and
     183    version are the names from the Discovery service.
     184
     185    Args:
     186      serviceName: string, name of the service.
     187      version: string, the version of the service.
     188      http: httplib2.Http, An instance of httplib2.Http or something that acts
     189        like it that HTTP requests will be made through.
     190      discoveryServiceUrl: string, a URI Template that points to the location of
     191        the discovery service. It should have two parameters {api} and
     192        {apiVersion} that when filled in produce an absolute URI to the discovery
     193        document for that service.
     194      developerKey: string, key obtained from
     195        https://code.google.com/apis/console.
     196      model: googleapiclient.Model, converts to and from the wire format.
     197      requestBuilder: googleapiclient.http.HttpRequest, encapsulator for an HTTP
     198        request.
     199      credentials: oauth2client.Credentials or
     200        google.auth.credentials.Credentials, credentials to be used for
     201        authentication.
     202      cache_discovery: Boolean, whether or not to cache the discovery doc.
     203      cache: googleapiclient.discovery_cache.base.CacheBase, an optional
     204        cache object for the discovery documents.
     205
     206    Returns:
     207      A Resource object with methods for interacting with the service.
     208    """
     209    params = {
     210        'api': serviceName,
     211        'apiVersion': version
     212        }
     213
     214    if http is None:
     215      discovery_http = build_http()
     216    else:
     217      discovery_http = http
     218
     219    for discovery_url in (discoveryServiceUrl, V2_DISCOVERY_URI,):
     220      requested_url = uritemplate.expand(discovery_url, params)
     221
     222      try:
     223        content = _retrieve_discovery_doc(
     224          requested_url, discovery_http, cache_discovery, cache, developerKey)
     225        return build_from_document(content, base=discovery_url, http=http,
     226            developerKey=developerKey, model=model, requestBuilder=requestBuilder,
     227            credentials=credentials)
     228      except HttpError as e:
     229        if e.resp.status == http_client.NOT_FOUND:
     230          continue
     231        else:
     232          raise e
     233
     234    raise UnknownApiNameOrVersion(
     235          "name: %s  version: %s" % (serviceName, version))
     236
     237
     238 -def _retrieve_discovery_doc(url, http, cache_discovery, cache=None,
     239                              developerKey=None):
     240    """Retrieves the discovery_doc from cache or the internet.
     241
     242    Args:
     243      url: string, the URL of the discovery document.
     244      http: httplib2.Http, An instance of httplib2.Http or something that acts
     245        like it through which HTTP requests will be made.
     246      cache_discovery: Boolean, whether or not to cache the discovery doc.
     247      cache: googleapiclient.discovery_cache.base.Cache, an optional cache
     248        object for the discovery documents.
     249
     250    Returns:
     251      A unicode string representation of the discovery document.
     252    """
     253    if cache_discovery:
     254      from . import discovery_cache
     255      from .discovery_cache import base
     256      if cache is None:
     257        cache = discovery_cache.autodetect()
     258      if cache:
     259        content = cache.get(url)
     260        if content:
     261          return content
     262
     263    actual_url = url
     264    # REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment
     265    # variable that contains the network address of the client sending the
     266    # request. If it exists then add that to the request for the discovery
     267    # document to avoid exceeding the quota on discovery requests.
     268    if 'REMOTE_ADDR' in os.environ:
     269      actual_url = _add_query_parameter(url, 'userIp', os.environ['REMOTE_ADDR'])
     270    if developerKey:
     271      actual_url = _add_query_parameter(url, 'key', developerKey)
     272    logger.info('URL being requested: GET %s', actual_url)
     273
     274    resp, content = http.request(actual_url)
     275
     276    if resp.status >= 400:
     277      raise HttpError(resp, content, uri=actual_url)
     278
     279    try:
     280      content = content.decode('utf-8')
     281    except AttributeError:
     282      pass
     283
     284    try:
     285      service = json.loads(content)
     286    except ValueError as e:
     287      logger.error('Failed to parse as JSON: ' + content)
     288      raise InvalidJsonError()
     289    if cache_discovery and cache:
     290      cache.set(url, content)
     291    return content
     292
     293
     294  @positional(1)
     295 -def build_from_document(
     296      service,
     297      base=None,
     298      future=None,
     299      http=None,
     300      developerKey=None,
     301      model=None,
     302      requestBuilder=HttpRequest,
     303      credentials=None):
     304    """Create a Resource for interacting with an API.
     305
     306    Same as `build()`, but constructs the Resource object from a discovery
     307    document that is it given, as opposed to retrieving one over HTTP.
     308
     309    Args:
     310      service: string or object, the JSON discovery document describing the API.
     311        The value passed in may either be the JSON string or the deserialized
     312        JSON.
     313      base: string, base URI for all HTTP requests, usually the discovery URI.
     314        This parameter is no longer used as rootUrl and servicePath are included
     315        within the discovery document. (deprecated)
     316      future: string, discovery document with future capabilities (deprecated).
     317      http: httplib2.Http, An instance of httplib2.Http or something that acts
     318        like it that HTTP requests will be made through.
     319      developerKey: string, Key for controlling API usage, generated
     320        from the API Console.
     321      model: Model class instance that serializes and de-serializes requests and
     322        responses.
     323      requestBuilder: Takes an http request and packages it up to be executed.
     324      credentials: oauth2client.Credentials or
     325        google.auth.credentials.Credentials, credentials to be used for
     326        authentication.
     327
     328    Returns:
     329      A Resource object with methods for interacting with the service.
     330    """
     331
     332    if http is not None and credentials is not None:
     333      raise ValueError('Arguments http and credentials are mutually exclusive.')
     334
     335    if isinstance(service, six.string_types):
     336      service = json.loads(service)
     337
     338    if  'rootUrl' not in service and (isinstance(http, (HttpMock,
     339                                                        HttpMockSequence))):
     340        logger.error("You are using HttpMock or HttpMockSequence without" +
     341                     "having the service discovery doc in cache. Try calling " +
     342                     "build() without mocking once first to populate the " +
     343                     "cache.")
     344        raise InvalidJsonError()
     345
     346    base = urljoin(service['rootUrl'], service['servicePath'])
     347    schema = Schemas(service)
     348
     349    # If the http client is not specified, then we must construct an http client
     350    # to make requests. If the service has scopes, then we also need to setup
     351    # authentication.
     352    if http is None:
     353      # Does the service require scopes?
     354      scopes = list(
     355        service.get('auth', {}).get('oauth2', {}).get('scopes', {}).keys())
     356
     357      # If so, then the we need to setup authentication if no developerKey is
     358      # specified.
     359      if scopes and not developerKey:
     360        # If the user didn't pass in credentials, attempt to acquire application
     361        # default credentials.
     362        if credentials is None:
     363          credentials = _auth.default_credentials()
     364
     365        # The credentials need to be scoped.
     366        credentials = _auth.with_scopes(credentials, scopes)
     367
     368      # If credentials are provided, create an authorized http instance;
     369      # otherwise, skip authentication.
     370      if credentials:
     371        http = _auth.authorized_http(credentials)
     372
     373      # If the service doesn't require scopes then there is no need for
     374      # authentication.
     375      else:
     376        http = build_http()
     377
     378    if model is None:
     379      features = service.get('features', [])
     380      model = JsonModel('dataWrapper' in features)
     381
     382    return Resource(http=http, baseUrl=base, model=model,
     383                    developerKey=developerKey, requestBuilder=requestBuilder,
     384                    resourceDesc=service, rootDesc=service, schema=schema)
     385
     386
     387 -def _cast(value, schema_type):
     388    """Convert value to a string based on JSON Schema type.
     389
     390    See http://tools.ietf.org/html/draft-zyp-json-schema-03 for more details on
     391    JSON Schema.
     392
     393    Args:
     394      value: any, the value to convert
     395      schema_type: string, the type that value should be interpreted as
     396
     397    Returns:
     398      A string representation of 'value' based on the schema_type.
     399    """
     400    if schema_type == 'string':
     401      if type(value) == type('') or type(value) == type(u''):
     402        return value
     403      else:
     404        return str(value)
     405    elif schema_type == 'integer':
     406      return str(int(value))
     407    elif schema_type == 'number':
     408      return str(float(value))
     409    elif schema_type == 'boolean':
     410      return str(bool(value)).lower()
     411    else:
     412      if type(value) == type('') or type(value) == type(u''):
     413        return value
     414      else:
     415        return str(value)
     416
     417
     418 -def _media_size_to_long(maxSize):
     419    """Convert a string media size, such as 10GB or 3TB into an integer.
     420
     421    Args:
     422      maxSize: string, size as a string, such as 2MB or 7GB.
     423
     424    Returns:
     425      The size as an integer value.
     426    """
     427    if len(maxSize) < 2:
     428      return 0
     429    units = maxSize[-2:].upper()
     430    bit_shift = _MEDIA_SIZE_BIT_SHIFTS.get(units)
     431    if bit_shift is not None:
     432      return int(maxSize[:-2]) << bit_shift
     433    else:
     434      return int(maxSize)
     435
     436
     437 -def _media_path_url_from_info(root_desc, path_url):
     438    """Creates an absolute media path URL.
     439
     440    Constructed using the API root URI and service path from the discovery
     441    document and the relative path for the API method.
     442
     443    Args:
     444      root_desc: Dictionary; the entire original deserialized discovery document.
     445      path_url: String; the relative URL for the API method. Relative to the API
     446          root, which is specified in the discovery document.
     447
     448    Returns:
     449      String; the absolute URI for media upload for the API method.
     450    """
     451    return '%(root)supload/%(service_path)s%(path)s' % {
     452        'root': root_desc['rootUrl'],
     453        'service_path': root_desc['servicePath'],
     454        'path': path_url,
     455    }
     456
     457
     458 -def _fix_up_parameters(method_desc, root_desc, http_method, schema):
     459    """Updates parameters of an API method with values specific to this library.
     460
     461    Specifically, adds whatever global parameters are specified by the API to the
     462    parameters for the individual method. Also adds parameters which don't
     463    appear in the discovery document, but are available to all discovery based
     464    APIs (these are listed in STACK_QUERY_PARAMETERS).
     465
     466    SIDE EFFECTS: This updates the parameters dictionary object in the method
     467    description.
     468
     469    Args:
     470      method_desc: Dictionary with metadata describing an API method. Value comes
     471          from the dictionary of methods stored in the 'methods' key in the
     472          deserialized discovery document.
     473      root_desc: Dictionary; the entire original deserialized discovery document.
     474      http_method: String; the HTTP method used to call the API method described
     475          in method_desc.
     476      schema: Object, mapping of schema names to schema descriptions.
     477
     478    Returns:
     479      The updated Dictionary stored in the 'parameters' key of the method
     480          description dictionary.
     481    """
     482    parameters = method_desc.setdefault('parameters', {})
     483
     484    # Add in the parameters common to all methods.
     485    for name, description in six.iteritems(root_desc.get('parameters', {})):
     486      parameters[name] = description
     487
     488    # Add in undocumented query parameters.
     489    for name in STACK_QUERY_PARAMETERS:
     490      parameters[name] = STACK_QUERY_PARAMETER_DEFAULT_VALUE.copy()
     491
     492    # Add 'body' (our own reserved word) to parameters if the method supports
     493    # a request payload.
     494    if http_method in HTTP_PAYLOAD_METHODS and 'request' in method_desc:
     495      body = BODY_PARAMETER_DEFAULT_VALUE.copy()
     496      body.update(method_desc['request'])
     497      # Make body optional for requests with no parameters.
     498      if not _methodProperties(method_desc, schema, 'request'):
     499        body['required'] = False
     500      parameters['body'] = body
     501
     502    return parameters
     503
     504
     505 -def _fix_up_media_upload(method_desc, root_desc, path_url, parameters):
     506    """Adds 'media_body' and 'media_mime_type' parameters if supported by method.
     507
     508    SIDE EFFECTS: If the method supports media upload and has a required body,
     509    sets body to be optional (required=False) instead. Also, if there is a
     510    'mediaUpload' in the method description, adds 'media_upload' key to
     511    parameters.
     512
     513    Args:
     514      method_desc: Dictionary with metadata describing an API method. Value comes
     515          from the dictionary of methods stored in the 'methods' key in the
     516          deserialized discovery document.
     517      root_desc: Dictionary; the entire original deserialized discovery document.
     518      path_url: String; the relative URL for the API method. Relative to the API
     519          root, which is specified in the discovery document.
     520      parameters: A dictionary describing method parameters for method described
     521          in method_desc.
     522
     523    Returns:
     524      Triple (accept, max_size, media_path_url) where:
     525        - accept is a list of strings representing what content types are
     526          accepted for media upload. Defaults to empty list if not in the
     527          discovery document.
     528        - max_size is a long representing the max size in bytes allowed for a
     529          media upload. Defaults to 0L if not in the discovery document.
     530        - media_path_url is a String; the absolute URI for media upload for the
     531          API method. Constructed using the API root URI and service path from
     532          the discovery document and the relative path for the API method. If
     533          media upload is not supported, this is None.
     534    """
     535    media_upload = method_desc.get('mediaUpload', {})
     536    accept = media_upload.get('accept', [])
     537    max_size = _media_size_to_long(media_upload.get('maxSize', ''))
     538    media_path_url = None
     539
     540    if media_upload:
     541      media_path_url = _media_path_url_from_info(root_desc, path_url)
     542      parameters['media_body'] = MEDIA_BODY_PARAMETER_DEFAULT_VALUE.copy()
     543      parameters['media_mime_type'] = MEDIA_MIME_TYPE_PARAMETER_DEFAULT_VALUE.copy()
     544      if 'body' in parameters:
     545        parameters['body']['required'] = False
     546
     547    return accept, max_size, media_path_url
     548
     549
     550 -def _fix_up_method_description(method_desc, root_desc, schema):
     551    """Updates a method description in a discovery document.
     552
     553    SIDE EFFECTS: Changes the parameters dictionary in the method description with
     554    extra parameters which are used locally.
     555
     556    Args:
     557      method_desc: Dictionary with metadata describing an API method. Value comes
     558          from the dictionary of methods stored in the 'methods' key in the
     559          deserialized discovery document.
     560      root_desc: Dictionary; the entire original deserialized discovery document.
     561      schema: Object, mapping of schema names to schema descriptions.
     562
     563    Returns:
     564      Tuple (path_url, http_method, method_id, accept, max_size, media_path_url)
     565      where:
     566        - path_url is a String; the relative URL for the API method. Relative to
     567          the API root, which is specified in the discovery document.
     568        - http_method is a String; the HTTP method used to call the API method
     569          described in the method description.
     570        - method_id is a String; the name of the RPC method associated with the
     571          API method, and is in the method description in the 'id' key.
     572        - accept is a list of strings representing what content types are
     573          accepted for media upload. Defaults to empty list if not in the
     574          discovery document.
     575        - max_size is a long representing the max size in bytes allowed for a
     576          media upload. Defaults to 0L if not in the discovery document.
     577        - media_path_url is a String; the absolute URI for media upload for the
     578          API method. Constructed using the API root URI and service path from
     579          the discovery document and the relative path for the API method. If
     580          media upload is not supported, this is None.
     581    """
     582    path_url = method_desc['path']
     583    http_method = method_desc['httpMethod']
     584    method_id = method_desc['id']
     585
     586    parameters = _fix_up_parameters(method_desc, root_desc, http_method, schema)
     587    # Order is important. `_fix_up_media_upload` needs `method_desc` to have a
     588    # 'parameters' key and needs to know if there is a 'body' parameter because it
     589    # also sets a 'media_body' parameter.
     590    accept, max_size, media_path_url = _fix_up_media_upload(
     591        method_desc, root_desc, path_url, parameters)
     592
     593    return path_url, http_method, method_id, accept, max_size, media_path_url
     594
     595
     596 -def _urljoin(base, url):
     597    """Custom urljoin replacement supporting : before / in url."""
     598    # In general, it's unsafe to simply join base and url. However, for
     599    # the case of discovery documents, we know:
     600    #  * base will never contain params, query, or fragment
     601    #  * url will never contain a scheme or net_loc.
     602    # In general, this means we can safely join on /; we just need to
     603    # ensure we end up with precisely one / joining base and url. The
     604    # exception here is the case of media uploads, where url will be an
     605    # absolute url.
     606    if url.startswith('http://') or url.startswith('https://'):
     607      return urljoin(base, url)
     608    new_base = base if base.endswith('/') else base + '/'
     609    new_url = url[1:] if url.startswith('/') else url
     610    return new_base + new_url
     611
     612
     613  # TODO(dhermes): Convert this class to ResourceMethod and make it callable
     614 -class ResourceMethodParameters(object):
     615    """Represents the parameters associated with a method.
     616
     617    Attributes:
     618      argmap: Map from method parameter name (string) to query parameter name
     619          (string).
     620      required_params: List of required parameters (represented by parameter
     621          name as string).
     622      repeated_params: List of repeated parameters (represented by parameter
     623          name as string).
     624      pattern_params: Map from method parameter name (string) to regular
     625          expression (as a string). If the pattern is set for a parameter, the
     626          value for that parameter must match the regular expression.
     627      query_params: List of parameters (represented by parameter name as string)
     628          that will be used in the query string.
     629      path_params: Set of parameters (represented by parameter name as string)
     630          that will be used in the base URL path.
     631      param_types: Map from method parameter name (string) to parameter type. Type
     632          can be any valid JSON schema type; valid values are 'any', 'array',
     633          'boolean', 'integer', 'number', 'object', or 'string'. Reference:
     634          http://tools.ietf.org/html/draft-zyp-json-schema-03#section-5.1
     635      enum_params: Map from method parameter name (string) to list of strings,
     636         where each list of strings is the list of acceptable enum values.
     637    """
     638
     639 -  def __init__(self, method_desc):
     640      """Constructor for ResourceMethodParameters.
     641
     642      Sets default values and defers to set_parameters to populate.
     643
     644      Args:
     645        method_desc: Dictionary with metadata describing an API method. Value
     646            comes from the dictionary of methods stored in the 'methods' key in
     647            the deserialized discovery document.
     648      """
     649      self.argmap = {}
     650      self.required_params = []
     651      self.repeated_params = []
     652      self.pattern_params = {}
     653      self.query_params = []
     654      # TODO(dhermes): Change path_params to a list if the extra URITEMPLATE
     655      #                parsing is gotten rid of.
     656      self.path_params = set()
     657      self.param_types = {}
     658      self.enum_params = {}
     659
     660      self.set_parameters(method_desc)
     661
     662 -  def set_parameters(self, method_desc):
     663      """Populates maps and lists based on method description.
     664
     665      Iterates through each parameter for the method and parses the values from
     666      the parameter dictionary.
     667
     668      Args:
     669        method_desc: Dictionary with metadata describing an API method. Value
     670            comes from the dictionary of methods stored in the 'methods' key in
     671            the deserialized discovery document.
     672      """
     673      for arg, desc in six.iteritems(method_desc.get('parameters', {})):
     674        param = key2param(arg)
     675        self.argmap[param] = arg
     676
     677        if desc.get('pattern'):
     678          self.pattern_params[param] = desc['pattern']
     679        if desc.get('enum'):
     680          self.enum_params[param] = desc['enum']
     681        if desc.get('required'):
     682          self.required_params.append(param)
     683        if desc.get('repeated'):
     684          self.repeated_params.append(param)
     685        if desc.get('location') == 'query':
     686          self.query_params.append(param)
     687        if desc.get('location') == 'path':
     688          self.path_params.add(param)
     689        self.param_types[param] = desc.get('type', 'string')
     690
     691      # TODO(dhermes): Determine if this is still necessary. Discovery based APIs
     692      #                should have all path parameters already marked with
     693      #                'location: path'.
     694      for match in URITEMPLATE.finditer(method_desc['path']):
     695        for namematch in VARNAME.finditer(match.group(0)):
     696          name = key2param(namematch.group(0))
     697          self.path_params.add(name)
     698          if name in self.query_params:
     699            self.query_params.remove(name)
     700
     701
     702 -def createMethod(methodName, methodDesc, rootDesc, schema):
     703    """Creates a method for attaching to a Resource.
     704
     705    Args:
     706      methodName: string, name of the method to use.
     707      methodDesc: object, fragment of deserialized discovery document that
     708        describes the method.
     709      rootDesc: object, the entire deserialized discovery document.
     710      schema: object, mapping of schema names to schema descriptions.
     711    """
     712    methodName = fix_method_name(methodName)
     713    (pathUrl, httpMethod, methodId, accept,
     714     maxSize, mediaPathUrl) = _fix_up_method_description(methodDesc, rootDesc, schema)
     715
     716    parameters = ResourceMethodParameters(methodDesc)
     717
     718    def method(self, **kwargs):
     719      # Don't bother with doc string, it will be over-written by createMethod.
     720
     721      for name in six.iterkeys(kwargs):
     722        if name not in parameters.argmap:
     723          raise TypeError('Got an unexpected keyword argument "%s"' % name)
     724
     725      # Remove args that have a value of None.
     726      keys = list(kwargs.keys())
     727      for name in keys:
     728        if kwargs[name] is None:
     729          del kwargs[name]
     730
     731      for name in parameters.required_params:
     732        if name not in kwargs:
     733          # temporary workaround for non-paging methods incorrectly requiring
     734          # page token parameter (cf. drive.changes.watch vs. drive.changes.list)
     735          if name not in _PAGE_TOKEN_NAMES or _findPageTokenName(
     736              _methodProperties(methodDesc, schema, 'response')):
     737            raise TypeError('Missing required parameter "%s"' % name)
     738
     739      for name, regex in six.iteritems(parameters.pattern_params):
     740        if name in kwargs:
     741          if isinstance(kwargs[name], six.string_types):
     742            pvalues = [kwargs[name]]
     743          else:
     744            pvalues = kwargs[name]
     745          for pvalue in pvalues:
     746            if re.match(regex, pvalue) is None:
     747              raise TypeError(
     748                  'Parameter "%s" value "%s" does not match the pattern "%s"' %
     749                  (name, pvalue, regex))
     750
     751      for name, enums in six.iteritems(parameters.enum_params):
     752        if name in kwargs:
     753          # We need to handle the case of a repeated enum
     754          # name differently, since we want to handle both
     755          # arg='value' and arg=['value1', 'value2']
     756          if (name in parameters.repeated_params and
     757              not isinstance(kwargs[name], six.string_types)):
     758            values = kwargs[name]
     759          else:
     760            values = [kwargs[name]]
     761          for value in values:
     762            if value not in enums:
     763              raise TypeError(
     764                  'Parameter "%s" value "%s" is not an allowed value in "%s"' %
     765                  (name, value, str(enums)))
     766
     767      actual_query_params = {}
     768      actual_path_params = {}
     769      for key, value in six.iteritems(kwargs):
     770        to_type = parameters.param_types.get(key, 'string')
     771        # For repeated parameters we cast each member of the list.
     772        if key in parameters.repeated_params and type(value) == type([]):
     773          cast_value = [_cast(x, to_type) for x in value]
     774        else:
     775          cast_value = _cast(value, to_type)
     776        if key in parameters.query_params:
     777          actual_query_params[parameters.argmap[key]] = cast_value
     778        if key in parameters.path_params:
     779          actual_path_params[parameters.argmap[key]] = cast_value
     780      body_value = kwargs.get('body', None)
     781      media_filename = kwargs.get('media_body', None)
     782      media_mime_type = kwargs.get('media_mime_type', None)
     783
     784      if self._developerKey:
     785        actual_query_params['key'] = self._developerKey
     786
     787      model = self._model
     788      if methodName.endswith('_media'):
     789        model = MediaModel()
     790      elif 'response' not in methodDesc:
     791        model = RawModel()
     792
     793      headers = {}
     794      headers, params, query, body = model.request(headers,
     795          actual_path_params, actual_query_params, body_value)
     796
     797      expanded_url = uritemplate.expand(pathUrl, params)
     798      url = _urljoin(self._baseUrl, expanded_url + query)
     799
     800      resumable = None
     801      multipart_boundary = ''
     802
     803      if media_filename:
     804        # Ensure we end up with a valid MediaUpload object.
     805        if isinstance(media_filename, six.string_types):
     806          if media_mime_type is None:
     807            logger.warning(
     808                'media_mime_type argument not specified: trying to auto-detect for %s',
     809                media_filename)
     810            media_mime_type, _ = mimetypes.guess_type(media_filename)
     811          if media_mime_type is None:
     812            raise UnknownFileType(media_filename)
     813          if not mimeparse.best_match([media_mime_type], ','.join(accept)):
     814            raise UnacceptableMimeTypeError(media_mime_type)
     815          media_upload = MediaFileUpload(media_filename,
     816                                         mimetype=media_mime_type)
     817        elif isinstance(media_filename, MediaUpload):
     818          media_upload = media_filename
     819        else:
     820          raise TypeError('media_filename must be str or MediaUpload.')
     821
     822        # Check the maxSize
     823        if media_upload.size() is not None and media_upload.size() > maxSize > 0:
     824          raise MediaUploadSizeError("Media larger than: %s" % maxSize)
     825
     826        # Use the media path uri for media uploads
     827        expanded_url = uritemplate.expand(mediaPathUrl, params)
     828        url = _urljoin(self._baseUrl, expanded_url + query)
     829        if media_upload.resumable():
     830          url = _add_query_parameter(url, 'uploadType', 'resumable')
     831
     832        if media_upload.resumable():
     833          # This is all we need to do for resumable, if the body exists it gets
     834          # sent in the first request, otherwise an empty body is sent.
     835          resumable = media_upload
     836        else:
     837          # A non-resumable upload
     838          if body is None:
     839            # This is a simple media upload
     840            headers['content-type'] = media_upload.mimetype()
     841            body = media_upload.getbytes(0, media_upload.size())
     842            url = _add_query_parameter(url, 'uploadType', 'media')
     843          else:
     844            # This is a multipart/related upload.
     845            msgRoot = MIMEMultipart('related')
     846            # msgRoot should not write out it's own headers
     847            setattr(msgRoot, '_write_headers', lambda self: None)
     848
     849            # attach the body as one part
     850            msg = MIMENonMultipart(*headers['content-type'].split('/'))
     851            msg.set_payload(body)
     852            msgRoot.attach(msg)
     853
     854            # attach the media as the second part
     855            msg = MIMENonMultipart(*media_upload.mimetype().split('/'))
     856            msg['Content-Transfer-Encoding'] = 'binary'
     857
     858            payload = media_upload.getbytes(0, media_upload.size())
     859            msg.set_payload(payload)
     860            msgRoot.attach(msg)
     861            # encode the body: note that we can't use `as_string`, because
     862            # it plays games with `From ` lines.
     863            fp = BytesIO()
     864            g = _BytesGenerator(fp, mangle_from_=False)
     865            g.flatten(msgRoot, unixfrom=False)
     866            body = fp.getvalue()
     867
     868            multipart_boundary = msgRoot.get_boundary()
     869            headers['content-type'] = ('multipart/related; '
     870                                       'boundary="%s"') % multipart_boundary
     871            url = _add_query_parameter(url, 'uploadType', 'multipart')
     872
     873      logger.info('URL being requested: %s %s' % (httpMethod,url))
     874      return self._requestBuilder(self._http,
     875                                  model.response,
     876                                  url,
     877                                  method=httpMethod,
     878                                  body=body,
     879                                  headers=headers,
     880                                  methodId=methodId,
     881                                  resumable=resumable)
     882
     883    docs = [methodDesc.get('description', DEFAULT_METHOD_DOC), '\n\n']
     884    if len(parameters.argmap) > 0:
     885      docs.append('Args:\n')
     886
     887    # Skip undocumented params and params common to all methods.
     888    skip_parameters = list(rootDesc.get('parameters', {}).keys())
     889    skip_parameters.extend(STACK_QUERY_PARAMETERS)
     890
     891    all_args = list(parameters.argmap.keys())
     892    args_ordered = [key2param(s) for s in methodDesc.get('parameterOrder', [])]
     893
     894    # Move body to the front of the line.
     895    if 'body' in all_args:
     896      args_ordered.append('body')
     897
     898    for name in all_args:
     899      if name not in args_ordered:
     900        args_ordered.append(name)
     901
     902    for arg in args_ordered:
     903      if arg in skip_parameters:
     904        continue
     905
     906      repeated = ''
     907      if arg in parameters.repeated_params:
     908        repeated = ' (repeated)'
     909      required = ''
     910      if arg in parameters.required_params:
     911        required = ' (required)'
     912      paramdesc = methodDesc['parameters'][parameters.argmap[arg]]
     913      paramdoc = paramdesc.get('description', 'A parameter')
     914      if '$ref' in paramdesc:
     915        docs.append(
     916            ('  %s: object, %s%s%s\n    The object takes the'
     917            ' form of:\n\n%s\n\n') % (arg, paramdoc, required, repeated,
     918              schema.prettyPrintByName(paramdesc['$ref'])))
     919      else:
     920        paramtype = paramdesc.get('type', 'string')
     921        docs.append('  %s: %s, %s%s%s\n' % (arg, paramtype, paramdoc, required,
     922                                            repeated))
     923      enum = paramdesc.get('enum', [])
     924      enumDesc = paramdesc.get('enumDescriptions', [])
     925      if enum and enumDesc:
     926        docs.append('    Allowed values\n')
     927        for (name, desc) in zip(enum, enumDesc):
     928          docs.append('      %s - %s\n' % (name, desc))
     929    if 'response' in methodDesc:
     930      if methodName.endswith('_media'):
     931        docs.append('\nReturns:\n  The media object as a string.\n\n    ')
     932      else:
     933        docs.append('\nReturns:\n  An object of the form:\n\n    ')
     934        docs.append(schema.prettyPrintSchema(methodDesc['response']))
     935
     936    setattr(method, '__doc__', ''.join(docs))
     937    return (methodName, method)
     938
     939
     940 -def createNextMethod(methodName,
     941                       pageTokenName='pageToken',
     942                       nextPageTokenName='nextPageToken',
     943                       isPageTokenParameter=True):
     944    """Creates any _next methods for attaching to a Resource.
     945
     946    The _next methods allow for easy iteration through list() responses.
     947
     948    Args:
     949      methodName: string, name of the method to use.
     950      pageTokenName: string, name of request page token field.
     951      nextPageTokenName: string, name of response page token field.
     952      isPageTokenParameter: Boolean, True if request page token is a query
     953          parameter, False if request page token is a field of the request body.
     954    """
     955    methodName = fix_method_name(methodName)
     956
     957    def methodNext(self, previous_request, previous_response):
     958      """Retrieves the next page of results.
     959
     960  Args:
     961    previous_request: The request for the previous page. (required)
     962    previous_response: The response from the request for the previous page. (required)
     963
     964  Returns:
     965    A request object that you can call 'execute()' on to request the next
     966    page. Returns None if there are no more items in the collection.
     967      """
     968      # Retrieve nextPageToken from previous_response
     969      # Use as pageToken in previous_request to create new request.
     970
     971      nextPageToken = previous_response.get(nextPageTokenName, None)
     972      if not nextPageToken:
     973        return None
     974
     975      request = copy.copy(previous_request)
     976
     977      if isPageTokenParameter:
     978          # Replace pageToken value in URI
     979          request.uri = _add_query_parameter(
     980              request.uri, pageTokenName, nextPageToken)
     981          logger.info('Next page request URL: %s %s' % (methodName, request.uri))
     982      else:
     983          # Replace pageToken value in request body
     984          model = self._model
     985          body = model.deserialize(request.body)
     986          body[pageTokenName] = nextPageToken
     987          request.body = model.serialize(body)
     988          logger.info('Next page request body: %s %s' % (methodName, body))
     989
     990      return request
     991
     992    return (methodName, methodNext)
     993
     994
     995 -class Resource(object):
     996    """A class for interacting with a resource."""
     997
     998 -  def __init__(self, http, baseUrl, model, requestBuilder, developerKey,
     999                 resourceDesc, rootDesc, schema):
    1000      """Build a Resource from the API description.
    1001
    1002      Args:
    1003        http: httplib2.Http, Object to make http requests with.
    1004        baseUrl: string, base URL for the API. All requests are relative to this
    1005            URI.
    1006        model: googleapiclient.Model, converts to and from the wire format.
    1007        requestBuilder: class or callable that instantiates an
    1008            googleapiclient.HttpRequest object.
    1009        developerKey: string, key obtained from
    1010            https://code.google.com/apis/console
    1011        resourceDesc: object, section of deserialized discovery document that
    1012            describes a resource. Note that the top level discovery document
    1013            is considered a resource.
    1014        rootDesc: object, the entire deserialized discovery document.
    1015        schema: object, mapping of schema names to schema descriptions.
    1016      """
    1017      self._dynamic_attrs = []
    1018
    1019      self._http = http
    1020      self._baseUrl = baseUrl
    1021      self._model = model
    1022      self._developerKey = developerKey
    1023      self._requestBuilder = requestBuilder
    1024      self._resourceDesc = resourceDesc
    1025      self._rootDesc = rootDesc
    1026      self._schema = schema
    1027
    1028      self._set_service_methods()
    1029
    1030 -  def _set_dynamic_attr(self, attr_name, value):
    1031      """Sets an instance attribute and tracks it in a list of dynamic attributes.
    1032
    1033      Args:
    1034        attr_name: string; The name of the attribute to be set
    1035        value: The value being set on the object and tracked in the dynamic cache.
    1036      """
    1037      self._dynamic_attrs.append(attr_name)
    1038      self.__dict__[attr_name] = value
    1039
    1040 -  def __getstate__(self):
    1041      """Trim the state down to something that can be pickled.
    1042
    1043      Uses the fact that the instance variable _dynamic_attrs holds attrs that
    1044      will be wiped and restored on pickle serialization.
    1045      """
    1046      state_dict = copy.copy(self.__dict__)
    1047      for dynamic_attr in self._dynamic_attrs:
    1048        del state_dict[dynamic_attr]
    1049      del state_dict['_dynamic_attrs']
    1050      return state_dict
    1051
    1052 -  def __setstate__(self, state):
    1053      """Reconstitute the state of the object from being pickled.
    1054
    1055      Uses the fact that the instance variable _dynamic_attrs holds attrs that
    1056      will be wiped and restored on pickle serialization.
    1057      """
    1058      self.__dict__.update(state)
    1059      self._dynamic_attrs = []
    1060      self._set_service_methods()
    1061
    1062 -  def _set_service_methods(self):
    1063      self._add_basic_methods(self._resourceDesc, self._rootDesc, self._schema)
    1064      self._add_nested_resources(self._resourceDesc, self._rootDesc, self._schema)
    1065      self._add_next_methods(self._resourceDesc, self._schema)
    1066
    1067 -  def _add_basic_methods(self, resourceDesc, rootDesc, schema):
    1068      # If this is the root Resource, add a new_batch_http_request() method.
    1069      if resourceDesc == rootDesc:
    1070        batch_uri = '%s%s' % (
    1071          rootDesc['rootUrl'], rootDesc.get('batchPath', 'batch'))
    1072        def new_batch_http_request(callback=None):
    1073          """Create a BatchHttpRequest object based on the discovery document.
    1074
    1075          Args:
    1076            callback: callable, A callback to be called for each response, of the
    1077              form callback(id, response, exception). The first parameter is the
    1078              request id, and the second is the deserialized response object. The
    1079              third is an apiclient.errors.HttpError exception object if an HTTP
    1080              error occurred while processing the request, or None if no error
    1081              occurred.
    1082
    1083          Returns:
    1084            A BatchHttpRequest object based on the discovery document.
    1085          """
    1086          return BatchHttpRequest(callback=callback, batch_uri=batch_uri)
    1087        self._set_dynamic_attr('new_batch_http_request', new_batch_http_request)
    1088
    1089      # Add basic methods to Resource
    1090      if 'methods' in resourceDesc:
    1091        for methodName, methodDesc in six.iteritems(resourceDesc['methods']):
    1092          fixedMethodName, method = createMethod(
    1093              methodName, methodDesc, rootDesc, schema)
    1094          self._set_dynamic_attr(fixedMethodName,
    1095                                 method.__get__(self, self.__class__))
    1096          # Add in _media methods. The functionality of the attached method will
    1097          # change when it sees that the method name ends in _media.
    1098          if methodDesc.get('supportsMediaDownload', False):
    1099            fixedMethodName, method = createMethod(
    1100                methodName + '_media', methodDesc, rootDesc, schema)
    1101            self._set_dynamic_attr(fixedMethodName,
    1102                                   method.__get__(self, self.__class__))
    1103
    1104 -  def _add_nested_resources(self, resourceDesc, rootDesc, schema):
    1105      # Add in nested resources
    1106      if 'resources' in resourceDesc:
    1107
    1108        def createResourceMethod(methodName, methodDesc):
    1109          """Create a method on the Resource to access a nested Resource.
    1110
    1111          Args:
    1112            methodName: string, name of the method to use.
    1113            methodDesc: object, fragment of deserialized discovery document that
    1114              describes the method.
    1115          """
    1116          methodName = fix_method_name(methodName)
    1117
    1118          def methodResource(self):
    1119            return Resource(http=self._http, baseUrl=self._baseUrl,
    1120                            model=self._model, developerKey=self._developerKey,
    1121                            requestBuilder=self._requestBuilder,
    1122                            resourceDesc=methodDesc, rootDesc=rootDesc,
    1123                            schema=schema)
    1124
    1125          setattr(methodResource, '__doc__', 'A collection resource.')
    1126          setattr(methodResource, '__is_resource__', True)
    1127
    1128          return (methodName, methodResource)
    1129
    1130        for methodName, methodDesc in six.iteritems(resourceDesc['resources']):
    1131          fixedMethodName, method = createResourceMethod(methodName, methodDesc)
    1132          self._set_dynamic_attr(fixedMethodName,
    1133                                 method.__get__(self, self.__class__))
    1134
    1135 -  def _add_next_methods(self, resourceDesc, schema):
    1136      # Add _next() methods if and only if one of the names 'pageToken' or
    1137      # 'nextPageToken' occurs among the fields of both the method's response
    1138      # type either the method's request (query parameters) or request body.
    1139      if 'methods' not in resourceDesc:
    1140        return
    1141      for methodName, methodDesc in six.iteritems(resourceDesc['methods']):
    1142        nextPageTokenName = _findPageTokenName(
    1143            _methodProperties(methodDesc, schema, 'response'))
    1144        if not nextPageTokenName:
    1145          continue
    1146        isPageTokenParameter = True
    1147        pageTokenName = _findPageTokenName(methodDesc.get('parameters', {}))
    1148        if not pageTokenName:
    1149          isPageTokenParameter = False
    1150          pageTokenName = _findPageTokenName(
    1151              _methodProperties(methodDesc, schema, 'request'))
    1152        if not pageTokenName:
    1153          continue
    1154        fixedMethodName, method = createNextMethod(
    1155            methodName + '_next', pageTokenName, nextPageTokenName,
    1156            isPageTokenParameter)
    1157        self._set_dynamic_attr(fixedMethodName,
    1158                               method.__get__(self, self.__class__))
    1159
    1160
    1161 -def _findPageTokenName(fields):
    1162    """Search field names for one like a page token.
    1163
    1164    Args:
    1165      fields: container of string, names of fields.
    1166
    1167    Returns:
    1168      First name that is either 'pageToken' or 'nextPageToken' if one exists,
    1169      otherwise None.
    1170    """
    1171    return next((tokenName for tokenName in _PAGE_TOKEN_NAMES
    1172                if tokenName in fields), None)
    1173
    1174 -def _methodProperties(methodDesc, schema, name):
    1175    """Get properties of a field in a method description.
    1176
    1177    Args:
    1178      methodDesc: object, fragment of deserialized discovery document that
    1179        describes the method.
    1180      schema: object, mapping of schema names to schema descriptions.
    1181      name: string, name of top-level field in method description.
    1182
    1183    Returns:
    1184      Object representing fragment of deserialized discovery document
    1185      corresponding to 'properties' field of object corresponding to named field
    1186      in method description, if it exists, otherwise empty dict.
    1187    """
    1188    desc = methodDesc.get(name, {})
    1189    if '$ref' in desc:
    1190      desc = schema.get(desc['$ref'], {})
    1191    return desc.get('properties', {})
    1192