googleapiclient.http

Package googleapiclient :: Module http |
[hide private]
[frames] | no frames]

# Source Code for Module googleapiclient.http

       1  # Copyright 2014 Google Inc. All Rights Reserved.
       2  #
       3  # Licensed under the Apache License, Version 2.0 (the "License");
       4  # you may not use this file except in compliance with the License.
       5  # You may obtain a copy of the License at
       6  #
       7  #      http://www.apache.org/licenses/LICENSE-2.0
       8  #
       9  # Unless required by applicable law or agreed to in writing, software
      10  # distributed under the License is distributed on an "AS IS" BASIS,
      11  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12  # See the License for the specific language governing permissions and
      13  # limitations under the License.
      14
      15  """Classes to encapsulate a single HTTP request.
      16
      17  The classes implement a command pattern, with every
      18  object supporting an execute() method that does the
      19  actual HTTP request.
      20  """
      21  from __future__ import absolute_import
      22  import six
      23  from six.moves import http_client
      24  from six.moves import range
      25
      26  __author__ = 'jcgregorio@google.com (Joe Gregorio)'
      27
      28  from six import BytesIO, StringIO
      29  from six.moves.urllib.parse import urlparse, urlunparse, quote, unquote
      30
      31  import base64
      32  import copy
      33  import gzip
      34  import httplib2
      35  import json
      36  import logging
      37  import mimetypes
      38  import os
      39  import random
      40  import socket
      41  import sys
      42  import time
      43  import uuid
      44
      45  # TODO(issue 221): Remove this conditional import jibbajabba.
      46  try:
      47    import ssl
      48  except ImportError:
      49    _ssl_SSLError = object()
      50  else:
      51    _ssl_SSLError = ssl.SSLError
      52
      53  from email.generator import Generator
      54  from email.mime.multipart import MIMEMultipart
      55  from email.mime.nonmultipart import MIMENonMultipart
      56  from email.parser import FeedParser
      57
      58  from googleapiclient import _helpers as util
      59
      60  from googleapiclient import _auth
      61  from googleapiclient.errors import BatchError
      62  from googleapiclient.errors import HttpError
      63  from googleapiclient.errors import InvalidChunkSizeError
      64  from googleapiclient.errors import ResumableUploadError
      65  from googleapiclient.errors import UnexpectedBodyError
      66  from googleapiclient.errors import UnexpectedMethodError
      67  from googleapiclient.model import JsonModel
      68
      69
      70  LOGGER = logging.getLogger(__name__)
      71
      72  DEFAULT_CHUNK_SIZE = 100*1024*1024
      73
      74  MAX_URI_LENGTH = 2048
      75
      76  MAX_BATCH_LIMIT = 1000
      77
      78  _TOO_MANY_REQUESTS = 429
      79
      80  DEFAULT_HTTP_TIMEOUT_SEC = 60
      81
      82  _LEGACY_BATCH_URI = 'https://www.googleapis.com/batch'
      83
      84
      85 -def _should_retry_response(resp_status, content):
      86    """Determines whether a response should be retried.
      87
      88    Args:
      89      resp_status: The response status received.
      90      content: The response content body.
      91
      92    Returns:
      93      True if the response should be retried, otherwise False.
      94    """
      95    # Retry on 5xx errors.
      96    if resp_status >= 500:
      97      return True
      98
      99    # Retry on 429 errors.
     100    if resp_status == _TOO_MANY_REQUESTS:
     101      return True
     102
     103    # For 403 errors, we have to check for the `reason` in the response to
     104    # determine if we should retry.
     105    if resp_status == six.moves.http_client.FORBIDDEN:
     106      # If there's no details about the 403 type, don't retry.
     107      if not content:
     108        return False
     109
     110      # Content is in JSON format.
     111      try:
     112        data = json.loads(content.decode('utf-8'))
     113        if isinstance(data, dict):
     114          reason = data['error']['errors'][0]['reason']
     115        else:
     116          reason = data[0]['error']['errors']['reason']
     117      except (UnicodeDecodeError, ValueError, KeyError):
     118        LOGGER.warning('Invalid JSON content from response: %s', content)
     119        return False
     120
     121      LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason)
     122
     123      # Only retry on rate limit related failures.
     124      if reason in ('userRateLimitExceeded', 'rateLimitExceeded', ):
     125        return True
     126
     127    # Everything else is a success or non-retriable so break.
     128    return False
     129
     130
     131 -def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args,
     132                     **kwargs):
     133    """Retries an HTTP request multiple times while handling errors.
     134
     135    If after all retries the request still fails, last error is either returned as
     136    return value (for HTTP 5xx errors) or thrown (for ssl.SSLError).
     137
     138    Args:
     139      http: Http object to be used to execute request.
     140      num_retries: Maximum number of retries.
     141      req_type: Type of the request (used for logging retries).
     142      sleep, rand: Functions to sleep for random time between retries.
     143      uri: URI to be requested.
     144      method: HTTP method to be used.
     145      args, kwargs: Additional arguments passed to http.request.
     146
     147    Returns:
     148      resp, content - Response from the http request (may be HTTP 5xx).
     149    """
     150    resp = None
     151    content = None
     152    for retry_num in range(num_retries + 1):
     153      if retry_num > 0:
     154        # Sleep before retrying.
     155        sleep_time = rand() * 2 ** retry_num
     156        LOGGER.warning(
     157            'Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s',
     158            sleep_time, retry_num, num_retries, req_type, method, uri,
     159            resp.status if resp else exception)
     160        sleep(sleep_time)
     161
     162      try:
     163        exception = None
     164        resp, content = http.request(uri, method, *args, **kwargs)
     165      # Retry on SSL errors and socket timeout errors.
     166      except _ssl_SSLError as ssl_error:
     167        exception = ssl_error
     168      except socket.timeout as socket_timeout:
     169        # It's important that this be before socket.error as it's a subclass
     170        # socket.timeout has no errorcode
     171        exception = socket_timeout
     172      except socket.error as socket_error:
     173        # errno's contents differ by platform, so we have to match by name.
     174        if socket.errno.errorcode.get(socket_error.errno) not in {
     175          'WSAETIMEDOUT', 'ETIMEDOUT', 'EPIPE', 'ECONNABORTED'}:
     176          raise
     177        exception = socket_error
     178      except httplib2.ServerNotFoundError as server_not_found_error:
     179        exception = server_not_found_error
     180
     181      if exception:
     182        if retry_num == num_retries:
     183          raise exception
     184        else:
     185          continue
     186
     187      if not _should_retry_response(resp.status, content):
     188        break
     189
     190    return resp, content
     191
     192
     193 -class MediaUploadProgress(object):
     194    """Status of a resumable upload."""
     195
     196 -  def __init__(self, resumable_progress, total_size):
     197      """Constructor.
     198
     199      Args:
     200        resumable_progress: int, bytes sent so far.
     201        total_size: int, total bytes in complete upload, or None if the total
     202          upload size isn't known ahead of time.
     203      """
     204      self.resumable_progress = resumable_progress
     205      self.total_size = total_size
     206
     207 -  def progress(self):
     208      """Percent of upload completed, as a float.
     209
     210      Returns:
     211        the percentage complete as a float, returning 0.0 if the total size of
     212        the upload is unknown.
     213      """
     214      if self.total_size is not None and self.total_size != 0:
     215        return float(self.resumable_progress) / float(self.total_size)
     216      else:
     217        return 0.0
     218
     219
     220 -class MediaDownloadProgress(object):
     221    """Status of a resumable download."""
     222
     223 -  def __init__(self, resumable_progress, total_size):
     224      """Constructor.
     225
     226      Args:
     227        resumable_progress: int, bytes received so far.
     228        total_size: int, total bytes in complete download.
     229      """
     230      self.resumable_progress = resumable_progress
     231      self.total_size = total_size
     232
     233 -  def progress(self):
     234      """Percent of download completed, as a float.
     235
     236      Returns:
     237        the percentage complete as a float, returning 0.0 if the total size of
     238        the download is unknown.
     239      """
     240      if self.total_size is not None and self.total_size != 0:
     241        return float(self.resumable_progress) / float(self.total_size)
     242      else:
     243        return 0.0
     244
     245
     246 -class MediaUpload(object):
     247    """Describes a media object to upload.
     248
     249    Base class that defines the interface of MediaUpload subclasses.
     250
     251    Note that subclasses of MediaUpload may allow you to control the chunksize
     252    when uploading a media object. It is important to keep the size of the chunk
     253    as large as possible to keep the upload efficient. Other factors may influence
     254    the size of the chunk you use, particularly if you are working in an
     255    environment where individual HTTP requests may have a hardcoded time limit,
     256    such as under certain classes of requests under Google App Engine.
     257
     258    Streams are io.Base compatible objects that support seek(). Some MediaUpload
     259    subclasses support using streams directly to upload data. Support for
     260    streaming may be indicated by a MediaUpload sub-class and if appropriate for a
     261    platform that stream will be used for uploading the media object. The support
     262    for streaming is indicated by has_stream() returning True. The stream() method
     263    should return an io.Base object that supports seek(). On platforms where the
     264    underlying httplib module supports streaming, for example Python 2.6 and
     265    later, the stream will be passed into the http library which will result in
     266    less memory being used and possibly faster uploads.
     267
     268    If you need to upload media that can't be uploaded using any of the existing
     269    MediaUpload sub-class then you can sub-class MediaUpload for your particular
     270    needs.
     271    """
     272
     273 -  def chunksize(self):
     274      """Chunk size for resumable uploads.
     275
     276      Returns:
     277        Chunk size in bytes.
     278      """
     279      raise NotImplementedError()
     280
     281 -  def mimetype(self):
     282      """Mime type of the body.
     283
     284      Returns:
     285        Mime type.
     286      """
     287      return 'application/octet-stream'
     288
     289 -  def size(self):
     290      """Size of upload.
     291
     292      Returns:
     293        Size of the body, or None of the size is unknown.
     294      """
     295      return None
     296
     297 -  def resumable(self):
     298      """Whether this upload is resumable.
     299
     300      Returns:
     301        True if resumable upload or False.
     302      """
     303      return False
     304
     305 -  def getbytes(self, begin, end):
     306      """Get bytes from the media.
     307
     308      Args:
     309        begin: int, offset from beginning of file.
     310        length: int, number of bytes to read, starting at begin.
     311
     312      Returns:
     313        A string of bytes read. May be shorter than length if EOF was reached
     314        first.
     315      """
     316      raise NotImplementedError()
     317
     318 -  def has_stream(self):
     319      """Does the underlying upload support a streaming interface.
     320
     321      Streaming means it is an io.IOBase subclass that supports seek, i.e.
     322      seekable() returns True.
     323
     324      Returns:
     325        True if the call to stream() will return an instance of a seekable io.Base
     326        subclass.
     327      """
     328      return False
     329
     330 -  def stream(self):
     331      """A stream interface to the data being uploaded.
     332
     333      Returns:
     334        The returned value is an io.IOBase subclass that supports seek, i.e.
     335        seekable() returns True.
     336      """
     337      raise NotImplementedError()
     338
     339    @util.positional(1)
     340 -  def _to_json(self, strip=None):
     341      """Utility function for creating a JSON representation of a MediaUpload.
     342
     343      Args:
     344        strip: array, An array of names of members to not include in the JSON.
     345
     346      Returns:
     347         string, a JSON representation of this instance, suitable to pass to
     348         from_json().
     349      """
     350      t = type(self)
     351      d = copy.copy(self.__dict__)
     352      if strip is not None:
     353        for member in strip:
     354          del d[member]
     355      d['_class'] = t.__name__
     356      d['_module'] = t.__module__
     357      return json.dumps(d)
     358
     359 -  def to_json(self):
     360      """Create a JSON representation of an instance of MediaUpload.
     361
     362      Returns:
     363         string, a JSON representation of this instance, suitable to pass to
     364         from_json().
     365      """
     366      return self._to_json()
     367
     368    @classmethod
     369 -  def new_from_json(cls, s):
     370      """Utility class method to instantiate a MediaUpload subclass from a JSON
     371      representation produced by to_json().
     372
     373      Args:
     374        s: string, JSON from to_json().
     375
     376      Returns:
     377        An instance of the subclass of MediaUpload that was serialized with
     378        to_json().
     379      """
     380      data = json.loads(s)
     381      # Find and call the right classmethod from_json() to restore the object.
     382      module = data['_module']
     383      m = __import__(module, fromlist=module.split('.')[:-1])
     384      kls = getattr(m, data['_class'])
     385      from_json = getattr(kls, 'from_json')
     386      return from_json(s)
     387
     388
     389 -class MediaIoBaseUpload(MediaUpload):
     390    """A MediaUpload for a io.Base objects.
     391
     392    Note that the Python file object is compatible with io.Base and can be used
     393    with this class also.
     394
     395      fh = BytesIO('...Some data to upload...')
     396      media = MediaIoBaseUpload(fh, mimetype='image/png',
     397        chunksize=1024*1024, resumable=True)
     398      farm.animals().insert(
     399          id='cow',
     400          name='cow.png',
     401          media_body=media).execute()
     402
     403    Depending on the platform you are working on, you may pass -1 as the
     404    chunksize, which indicates that the entire file should be uploaded in a single
     405    request. If the underlying platform supports streams, such as Python 2.6 or
     406    later, then this can be very efficient as it avoids multiple connections, and
     407    also avoids loading the entire file into memory before sending it. Note that
     408    Google App Engine has a 5MB limit on request size, so you should never set
     409    your chunksize larger than 5MB, or to -1.
     410    """
     411
     412    @util.positional(3)
     413 -  def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE,
     414        resumable=False):
     415      """Constructor.
     416
     417      Args:
     418        fd: io.Base or file object, The source of the bytes to upload. MUST be
     419          opened in blocking mode, do not use streams opened in non-blocking mode.
     420          The given stream must be seekable, that is, it must be able to call
     421          seek() on fd.
     422        mimetype: string, Mime-type of the file.
     423        chunksize: int, File will be uploaded in chunks of this many bytes. Only
     424          used if resumable=True. Pass in a value of -1 if the file is to be
     425          uploaded as a single chunk. Note that Google App Engine has a 5MB limit
     426          on request size, so you should never set your chunksize larger than 5MB,
     427          or to -1.
     428        resumable: bool, True if this is a resumable upload. False means upload
     429          in a single request.
     430      """
     431      super(MediaIoBaseUpload, self).__init__()
     432      self._fd = fd
     433      self._mimetype = mimetype
     434      if not (chunksize == -1 or chunksize > 0):
     435        raise InvalidChunkSizeError()
     436      self._chunksize = chunksize
     437      self._resumable = resumable
     438
     439      self._fd.seek(0, os.SEEK_END)
     440      self._size = self._fd.tell()
     441
     442 -  def chunksize(self):
     443      """Chunk size for resumable uploads.
     444
     445      Returns:
     446        Chunk size in bytes.
     447      """
     448      return self._chunksize
     449
     450 -  def mimetype(self):
     451      """Mime type of the body.
     452
     453      Returns:
     454        Mime type.
     455      """
     456      return self._mimetype
     457
     458 -  def size(self):
     459      """Size of upload.
     460
     461      Returns:
     462        Size of the body, or None of the size is unknown.
     463      """
     464      return self._size
     465
     466 -  def resumable(self):
     467      """Whether this upload is resumable.
     468
     469      Returns:
     470        True if resumable upload or False.
     471      """
     472      return self._resumable
     473
     474 -  def getbytes(self, begin, length):
     475      """Get bytes from the media.
     476
     477      Args:
     478        begin: int, offset from beginning of file.
     479        length: int, number of bytes to read, starting at begin.
     480
     481      Returns:
     482        A string of bytes read. May be shorted than length if EOF was reached
     483        first.
     484      """
     485      self._fd.seek(begin)
     486      return self._fd.read(length)
     487
     488 -  def has_stream(self):
     489      """Does the underlying upload support a streaming interface.
     490
     491      Streaming means it is an io.IOBase subclass that supports seek, i.e.
     492      seekable() returns True.
     493
     494      Returns:
     495        True if the call to stream() will return an instance of a seekable io.Base
     496        subclass.
     497      """
     498      return True
     499
     500 -  def stream(self):
     501      """A stream interface to the data being uploaded.
     502
     503      Returns:
     504        The returned value is an io.IOBase subclass that supports seek, i.e.
     505        seekable() returns True.
     506      """
     507      return self._fd
     508
     509 -  def to_json(self):
     510      """This upload type is not serializable."""
     511      raise NotImplementedError('MediaIoBaseUpload is not serializable.')
     512
     513
     514 -class MediaFileUpload(MediaIoBaseUpload):
     515    """A MediaUpload for a file.
     516
     517    Construct a MediaFileUpload and pass as the media_body parameter of the
     518    method. For example, if we had a service that allowed uploading images:
     519
     520      media = MediaFileUpload('cow.png', mimetype='image/png',
     521        chunksize=1024*1024, resumable=True)
     522      farm.animals().insert(
     523          id='cow',
     524          name='cow.png',
     525          media_body=media).execute()
     526
     527    Depending on the platform you are working on, you may pass -1 as the
     528    chunksize, which indicates that the entire file should be uploaded in a single
     529    request. If the underlying platform supports streams, such as Python 2.6 or
     530    later, then this can be very efficient as it avoids multiple connections, and
     531    also avoids loading the entire file into memory before sending it. Note that
     532    Google App Engine has a 5MB limit on request size, so you should never set
     533    your chunksize larger than 5MB, or to -1.
     534    """
     535
     536    @util.positional(2)
     537 -  def __init__(self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE,
     538                 resumable=False):
     539      """Constructor.
     540
     541      Args:
     542        filename: string, Name of the file.
     543        mimetype: string, Mime-type of the file. If None then a mime-type will be
     544          guessed from the file extension.
     545        chunksize: int, File will be uploaded in chunks of this many bytes. Only
     546          used if resumable=True. Pass in a value of -1 if the file is to be
     547          uploaded in a single chunk. Note that Google App Engine has a 5MB limit
     548          on request size, so you should never set your chunksize larger than 5MB,
     549          or to -1.
     550        resumable: bool, True if this is a resumable upload. False means upload
     551          in a single request.
     552      """
     553      self._filename = filename
     554      fd = open(self._filename, 'rb')
     555      if mimetype is None:
     556        # No mimetype provided, make a guess.
     557        mimetype, _ = mimetypes.guess_type(filename)
     558        if mimetype is None:
     559          # Guess failed, use octet-stream.
     560          mimetype = 'application/octet-stream'
     561      super(MediaFileUpload, self).__init__(fd, mimetype, chunksize=chunksize,
     562                                            resumable=resumable)
     563
     564 -  def to_json(self):
     565      """Creating a JSON representation of an instance of MediaFileUpload.
     566
     567      Returns:
     568         string, a JSON representation of this instance, suitable to pass to
     569         from_json().
     570      """
     571      return self._to_json(strip=['_fd'])
     572
     573    @staticmethod
     574 -  def from_json(s):
     575      d = json.loads(s)
     576      return MediaFileUpload(d['_filename'], mimetype=d['_mimetype'],
     577                             chunksize=d['_chunksize'], resumable=d['_resumable'])
     578
     579
     580 -class MediaInMemoryUpload(MediaIoBaseUpload):
     581    """MediaUpload for a chunk of bytes.
     582
     583    DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for
     584    the stream.
     585    """
     586
     587    @util.positional(2)
     588 -  def __init__(self, body, mimetype='application/octet-stream',
     589                 chunksize=DEFAULT_CHUNK_SIZE, resumable=False):
     590      """Create a new MediaInMemoryUpload.
     591
     592    DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for
     593    the stream.
     594
     595    Args:
     596      body: string, Bytes of body content.
     597      mimetype: string, Mime-type of the file or default of
     598        'application/octet-stream'.
     599      chunksize: int, File will be uploaded in chunks of this many bytes. Only
     600        used if resumable=True.
     601      resumable: bool, True if this is a resumable upload. False means upload
     602        in a single request.
     603      """
     604      fd = BytesIO(body)
     605      super(MediaInMemoryUpload, self).__init__(fd, mimetype, chunksize=chunksize,
     606                                                resumable=resumable)
     607
     608
     609 -class MediaIoBaseDownload(object):
     610    """"Download media resources.
     611
     612    Note that the Python file object is compatible with io.Base and can be used
     613    with this class also.
     614
     615
     616    Example:
     617      request = farms.animals().get_media(id='cow')
     618      fh = io.FileIO('cow.png', mode='wb')
     619      downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024)
     620
     621      done = False
     622      while done is False:
     623        status, done = downloader.next_chunk()
     624        if status:
     625          print "Download %d%%." % int(status.progress() * 100)
     626      print "Download Complete!"
     627    """
     628
     629    @util.positional(3)
     630 -  def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE):
     631      """Constructor.
     632
     633      Args:
     634        fd: io.Base or file object, The stream in which to write the downloaded
     635          bytes.
     636        request: googleapiclient.http.HttpRequest, the media request to perform in
     637          chunks.
     638        chunksize: int, File will be downloaded in chunks of this many bytes.
     639      """
     640      self._fd = fd
     641      self._request = request
     642      self._uri = request.uri
     643      self._chunksize = chunksize
     644      self._progress = 0
     645      self._total_size = None
     646      self._done = False
     647
     648      # Stubs for testing.
     649      self._sleep = time.sleep
     650      self._rand = random.random
     651
     652      self._headers = {}
     653      for k, v in six.iteritems(request.headers):
     654        # allow users to supply custom headers by setting them on the request
     655        # but strip out the ones that are set by default on requests generated by
     656        # API methods like Drive's files().get(fileId=...)
     657        if not k.lower() in ('accept', 'accept-encoding', 'user-agent'):
     658          self._headers[k] = v
     659
     660    @util.positional(1)
     661 -  def next_chunk(self, num_retries=0):
     662      """Get the next chunk of the download.
     663
     664      Args:
     665        num_retries: Integer, number of times to retry with randomized
     666              exponential backoff. If all retries fail, the raised HttpError
     667              represents the last request. If zero (default), we attempt the
     668              request only once.
     669
     670      Returns:
     671        (status, done): (MediaDownloadProgress, boolean)
     672           The value of 'done' will be True when the media has been fully
     673           downloaded or the total size of the media is unknown.
     674
     675      Raises:
     676        googleapiclient.errors.HttpError if the response was not a 2xx.
     677        httplib2.HttpLib2Error if a transport error has occured.
     678      """
     679      headers = self._headers.copy()
     680      headers['range'] = 'bytes=%d-%d' % (
     681              self._progress, self._progress + self._chunksize)
     682      http = self._request.http
     683
     684      resp, content = _retry_request(
     685          http, num_retries, 'media download', self._sleep, self._rand, self._uri,
     686          'GET', headers=headers)
     687
     688      if resp.status in [200, 206]:
     689        if 'content-location' in resp and resp['content-location'] != self._uri:
     690          self._uri = resp['content-location']
     691        self._progress += len(content)
     692        self._fd.write(content)
     693
     694        if 'content-range' in resp:
     695          content_range = resp['content-range']
     696          length = content_range.rsplit('/', 1)[1]
     697          self._total_size = int(length)
     698        elif 'content-length' in resp:
     699          self._total_size = int(resp['content-length'])
     700
     701        if self._total_size is None or self._progress == self._total_size:
     702          self._done = True
     703        return MediaDownloadProgress(self._progress, self._total_size), self._done
     704      else:
     705        raise HttpError(resp, content, uri=self._uri)
     706
     707
     708 -class _StreamSlice(object):
     709    """Truncated stream.
     710
     711    Takes a stream and presents a stream that is a slice of the original stream.
     712    This is used when uploading media in chunks. In later versions of Python a
     713    stream can be passed to httplib in place of the string of data to send. The
     714    problem is that httplib just blindly reads to the end of the stream. This
     715    wrapper presents a virtual stream that only reads to the end of the chunk.
     716    """
     717
     718 -  def __init__(self, stream, begin, chunksize):
     719      """Constructor.
     720
     721      Args:
     722        stream: (io.Base, file object), the stream to wrap.
     723        begin: int, the seek position the chunk begins at.
     724        chunksize: int, the size of the chunk.
     725      """
     726      self._stream = stream
     727      self._begin = begin
     728      self._chunksize = chunksize
     729      self._stream.seek(begin)
     730
     731 -  def read(self, n=-1):
     732      """Read n bytes.
     733
     734      Args:
     735        n, int, the number of bytes to read.
     736
     737      Returns:
     738        A string of length 'n', or less if EOF is reached.
     739      """
     740      # The data left available to read sits in [cur, end)
     741      cur = self._stream.tell()
     742      end = self._begin + self._chunksize
     743      if n == -1 or cur + n > end:
     744        n = end - cur
     745      return self._stream.read(n)
     746
     747
     748 -class HttpRequest(object):
     749    """Encapsulates a single HTTP request."""
     750
     751    @util.positional(4)
     752 -  def __init__(self, http, postproc, uri,
     753                 method='GET',
     754                 body=None,
     755                 headers=None,
     756                 methodId=None,
     757                 resumable=None):
     758      """Constructor for an HttpRequest.
     759
     760      Args:
     761        http: httplib2.Http, the transport object to use to make a request
     762        postproc: callable, called on the HTTP response and content to transform
     763                  it into a data object before returning, or raising an exception
     764                  on an error.
     765        uri: string, the absolute URI to send the request to
     766        method: string, the HTTP method to use
     767        body: string, the request body of the HTTP request,
     768        headers: dict, the HTTP request headers
     769        methodId: string, a unique identifier for the API method being called.
     770        resumable: MediaUpload, None if this is not a resumbale request.
     771      """
     772      self.uri = uri
     773      self.method = method
     774      self.body = body
     775      self.headers = headers or {}
     776      self.methodId = methodId
     777      self.http = http
     778      self.postproc = postproc
     779      self.resumable = resumable
     780      self.response_callbacks = []
     781      self._in_error_state = False
     782
     783      # The size of the non-media part of the request.
     784      self.body_size = len(self.body or '')
     785
     786      # The resumable URI to send chunks to.
     787      self.resumable_uri = None
     788
     789      # The bytes that have been uploaded.
     790      self.resumable_progress = 0
     791
     792      # Stubs for testing.
     793      self._rand = random.random
     794      self._sleep = time.sleep
     795
     796    @util.positional(1)
     797 -  def execute(self, http=None, num_retries=0):
     798      """Execute the request.
     799
     800      Args:
     801        http: httplib2.Http, an http object to be used in place of the
     802              one the HttpRequest request object was constructed with.
     803        num_retries: Integer, number of times to retry with randomized
     804              exponential backoff. If all retries fail, the raised HttpError
     805              represents the last request. If zero (default), we attempt the
     806              request only once.
     807
     808      Returns:
     809        A deserialized object model of the response body as determined
     810        by the postproc.
     811
     812      Raises:
     813        googleapiclient.errors.HttpError if the response was not a 2xx.
     814        httplib2.HttpLib2Error if a transport error has occured.
     815      """
     816      if http is None:
     817        http = self.http
     818
     819      if self.resumable:
     820        body = None
     821        while body is None:
     822          _, body = self.next_chunk(http=http, num_retries=num_retries)
     823        return body
     824
     825      # Non-resumable case.
     826
     827      if 'content-length' not in self.headers:
     828        self.headers['content-length'] = str(self.body_size)
     829      # If the request URI is too long then turn it into a POST request.
     830      # Assume that a GET request never contains a request body.
     831      if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET':
     832        self.method = 'POST'
     833        self.headers['x-http-method-override'] = 'GET'
     834        self.headers['content-type'] = 'application/x-www-form-urlencoded'
     835        parsed = urlparse(self.uri)
     836        self.uri = urlunparse(
     837            (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None,
     838             None)
     839            )
     840        self.body = parsed.query
     841        self.headers['content-length'] = str(len(self.body))
     842
     843      # Handle retries for server-side errors.
     844      resp, content = _retry_request(
     845            http, num_retries, 'request', self._sleep, self._rand, str(self.uri),
     846            method=str(self.method), body=self.body, headers=self.headers)
     847
     848      for callback in self.response_callbacks:
     849        callback(resp)
     850      if resp.status >= 300:
     851        raise HttpError(resp, content, uri=self.uri)
     852      return self.postproc(resp, content)
     853
     854    @util.positional(2)
     855 -  def add_response_callback(self, cb):
     856      """add_response_headers_callback
     857
     858      Args:
     859        cb: Callback to be called on receiving the response headers, of signature:
     860
     861        def cb(resp):
     862          # Where resp is an instance of httplib2.Response
     863      """
     864      self.response_callbacks.append(cb)
     865
     866    @util.positional(1)
     867 -  def next_chunk(self, http=None, num_retries=0):
     868      """Execute the next step of a resumable upload.
     869
     870      Can only be used if the method being executed supports media uploads and
     871      the MediaUpload object passed in was flagged as using resumable upload.
     872
     873      Example:
     874
     875        media = MediaFileUpload('cow.png', mimetype='image/png',
     876                                chunksize=1000, resumable=True)
     877        request = farm.animals().insert(
     878            id='cow',
     879            name='cow.png',
     880            media_body=media)
     881
     882        response = None
     883        while response is None:
     884          status, response = request.next_chunk()
     885          if status:
     886            print "Upload %d%% complete." % int(status.progress() * 100)
     887
     888
     889      Args:
     890        http: httplib2.Http, an http object to be used in place of the
     891              one the HttpRequest request object was constructed with.
     892        num_retries: Integer, number of times to retry with randomized
     893              exponential backoff. If all retries fail, the raised HttpError
     894              represents the last request. If zero (default), we attempt the
     895              request only once.
     896
     897      Returns:
     898        (status, body): (ResumableMediaStatus, object)
     899           The body will be None until the resumable media is fully uploaded.
     900
     901      Raises:
     902        googleapiclient.errors.HttpError if the response was not a 2xx.
     903        httplib2.HttpLib2Error if a transport error has occured.
     904      """
     905      if http is None:
     906        http = self.http
     907
     908      if self.resumable.size() is None:
     909        size = '*'
     910      else:
     911        size = str(self.resumable.size())
     912
     913      if self.resumable_uri is None:
     914        start_headers = copy.copy(self.headers)
     915        start_headers['X-Upload-Content-Type'] = self.resumable.mimetype()
     916        if size != '*':
     917          start_headers['X-Upload-Content-Length'] = size
     918        start_headers['content-length'] = str(self.body_size)
     919
     920        resp, content = _retry_request(
     921            http, num_retries, 'resumable URI request', self._sleep, self._rand,
     922            self.uri, method=self.method, body=self.body, headers=start_headers)
     923
     924        if resp.status == 200 and 'location' in resp:
     925          self.resumable_uri = resp['location']
     926        else:
     927          raise ResumableUploadError(resp, content)
     928      elif self._in_error_state:
     929        # If we are in an error state then query the server for current state of
     930        # the upload by sending an empty PUT and reading the 'range' header in
     931        # the response.
     932        headers = {
     933            'Content-Range': 'bytes */%s' % size,
     934            'content-length': '0'
     935            }
     936        resp, content = http.request(self.resumable_uri, 'PUT',
     937                                     headers=headers)
     938        status, body = self._process_response(resp, content)
     939        if body:
     940          # The upload was complete.
     941          return (status, body)
     942
     943      if self.resumable.has_stream():
     944        data = self.resumable.stream()
     945        if self.resumable.chunksize() == -1:
     946          data.seek(self.resumable_progress)
     947          chunk_end = self.resumable.size() - self.resumable_progress - 1
     948        else:
     949          # Doing chunking with a stream, so wrap a slice of the stream.
     950          data = _StreamSlice(data, self.resumable_progress,
     951                              self.resumable.chunksize())
     952          chunk_end = min(
     953              self.resumable_progress + self.resumable.chunksize() - 1,
     954              self.resumable.size() - 1)
     955      else:
     956        data = self.resumable.getbytes(
     957            self.resumable_progress, self.resumable.chunksize())
     958
     959        # A short read implies that we are at EOF, so finish the upload.
     960        if len(data) < self.resumable.chunksize():
     961          size = str(self.resumable_progress + len(data))
     962
     963        chunk_end = self.resumable_progress + len(data) - 1
     964
     965      headers = {
     966          'Content-Range': 'bytes %d-%d/%s' % (
     967              self.resumable_progress, chunk_end, size),
     968          # Must set the content-length header here because httplib can't
     969          # calculate the size when working with _StreamSlice.
     970          'Content-Length': str(chunk_end - self.resumable_progress + 1)
     971          }
     972
     973      for retry_num in range(num_retries + 1):
     974        if retry_num > 0:
     975          self._sleep(self._rand() * 2**retry_num)
     976          LOGGER.warning(
     977              'Retry #%d for media upload: %s %s, following status: %d'
     978              % (retry_num, self.method, self.uri, resp.status))
     979
     980        try:
     981          resp, content = http.request(self.resumable_uri, method='PUT',
     982                                       body=data,
     983                                       headers=headers)
     984        except:
     985          self._in_error_state = True
     986          raise
     987        if not _should_retry_response(resp.status, content):
     988          break
     989
     990      return self._process_response(resp, content)
     991
     992 -  def _process_response(self, resp, content):
     993      """Process the response from a single chunk upload.
     994
     995      Args:
     996        resp: httplib2.Response, the response object.
     997        content: string, the content of the response.
     998
     999      Returns:
    1000        (status, body): (ResumableMediaStatus, object)
    1001           The body will be None until the resumable media is fully uploaded.
    1002
    1003      Raises:
    1004        googleapiclient.errors.HttpError if the response was not a 2xx or a 308.
    1005      """
    1006      if resp.status in [200, 201]:
    1007        self._in_error_state = False
    1008        return None, self.postproc(resp, content)
    1009      elif resp.status == 308:
    1010        self._in_error_state = False
    1011        # A "308 Resume Incomplete" indicates we are not done.
    1012        try:
    1013          self.resumable_progress = int(resp['range'].split('-')[1]) + 1
    1014        except KeyError:
    1015          # If resp doesn't contain range header, resumable progress is 0
    1016          self.resumable_progress = 0
    1017        if 'location' in resp:
    1018          self.resumable_uri = resp['location']
    1019      else:
    1020        self._in_error_state = True
    1021        raise HttpError(resp, content, uri=self.uri)
    1022
    1023      return (MediaUploadProgress(self.resumable_progress, self.resumable.size()),
    1024              None)
    1025
    1026 -  def to_json(self):
    1027      """Returns a JSON representation of the HttpRequest."""
    1028      d = copy.copy(self.__dict__)
    1029      if d['resumable'] is not None:
    1030        d['resumable'] = self.resumable.to_json()
    1031      del d['http']
    1032      del d['postproc']
    1033      del d['_sleep']
    1034      del d['_rand']
    1035
    1036      return json.dumps(d)
    1037
    1038    @staticmethod
    1039 -  def from_json(s, http, postproc):
    1040      """Returns an HttpRequest populated with info from a JSON object."""
    1041      d = json.loads(s)
    1042      if d['resumable'] is not None:
    1043        d['resumable'] = MediaUpload.new_from_json(d['resumable'])
    1044      return HttpRequest(
    1045          http,
    1046          postproc,
    1047          uri=d['uri'],
    1048          method=d['method'],
    1049          body=d['body'],
    1050          headers=d['headers'],
    1051          methodId=d['methodId'],
    1052          resumable=d['resumable'])
    1053
    1054
    1055 -class BatchHttpRequest(object):
    1056    """Batches multiple HttpRequest objects into a single HTTP request.
    1057
    1058    Example:
    1059      from googleapiclient.http import BatchHttpRequest
    1060
    1061      def list_animals(request_id, response, exception):
    1062        \"\"\"Do something with the animals list response.\"\"\"
    1063        if exception is not None:
    1064          # Do something with the exception.
    1065          pass
    1066        else:
    1067          # Do something with the response.
    1068          pass
    1069
    1070      def list_farmers(request_id, response, exception):
    1071        \"\"\"Do something with the farmers list response.\"\"\"
    1072        if exception is not None:
    1073          # Do something with the exception.
    1074          pass
    1075        else:
    1076          # Do something with the response.
    1077          pass
    1078
    1079      service = build('farm', 'v2')
    1080
    1081      batch = BatchHttpRequest()
    1082
    1083      batch.add(service.animals().list(), list_animals)
    1084      batch.add(service.farmers().list(), list_farmers)
    1085      batch.execute(http=http)
    1086    """
    1087
    1088    @util.positional(1)
    1089 -  def __init__(self, callback=None, batch_uri=None):
    1090      """Constructor for a BatchHttpRequest.
    1091
    1092      Args:
    1093        callback: callable, A callback to be called for each response, of the
    1094          form callback(id, response, exception). The first parameter is the
    1095          request id, and the second is the deserialized response object. The
    1096          third is an googleapiclient.errors.HttpError exception object if an HTTP error
    1097          occurred while processing the request, or None if no error occurred.
    1098        batch_uri: string, URI to send batch requests to.
    1099      """
    1100      if batch_uri is None:
    1101        batch_uri = _LEGACY_BATCH_URI
    1102
    1103      if batch_uri == _LEGACY_BATCH_URI:
    1104        LOGGER.warn(
    1105          "You have constructed a BatchHttpRequest using the legacy batch "
    1106          "endpoint %s. This endpoint will be turned down on March 25, 2019. "
    1107          "Please provide the API-specific endpoint or use "
    1108          "service.new_batch_http_request(). For more details see "
    1109          "https://developers.googleblog.com/2018/03/discontinuing-support-for-json-rpc-and.html"
    1110          "and https://developers.google.com/api-client-library/python/guide/batch.",
    1111          _LEGACY_BATCH_URI)
    1112      self._batch_uri = batch_uri
    1113
    1114      # Global callback to be called for each individual response in the batch.
    1115      self._callback = callback
    1116
    1117      # A map from id to request.
    1118      self._requests = {}
    1119
    1120      # A map from id to callback.
    1121      self._callbacks = {}
    1122
    1123      # List of request ids, in the order in which they were added.
    1124      self._order = []
    1125
    1126      # The last auto generated id.
    1127      self._last_auto_id = 0
    1128
    1129      # Unique ID on which to base the Content-ID headers.
    1130      self._base_id = None
    1131
    1132      # A map from request id to (httplib2.Response, content) response pairs
    1133      self._responses = {}
    1134
    1135      # A map of id(Credentials) that have been refreshed.
    1136      self._refreshed_credentials = {}
    1137
    1138 -  def _refresh_and_apply_credentials(self, request, http):
    1139      """Refresh the credentials and apply to the request.
    1140
    1141      Args:
    1142        request: HttpRequest, the request.
    1143        http: httplib2.Http, the global http object for the batch.
    1144      """
    1145      # For the credentials to refresh, but only once per refresh_token
    1146      # If there is no http per the request then refresh the http passed in
    1147      # via execute()
    1148      creds = None
    1149      request_credentials = False
    1150
    1151      if request.http is not None:
    1152        creds = _auth.get_credentials_from_http(request.http)
    1153        request_credentials = True
    1154
    1155      if creds is None and http is not None:
    1156        creds = _auth.get_credentials_from_http(http)
    1157
    1158      if creds is not None:
    1159        if id(creds) not in self._refreshed_credentials:
    1160          _auth.refresh_credentials(creds)
    1161          self._refreshed_credentials[id(creds)] = 1
    1162
    1163      # Only apply the credentials if we are using the http object passed in,
    1164      # otherwise apply() will get called during _serialize_request().
    1165      if request.http is None or not request_credentials:
    1166        _auth.apply_credentials(creds, request.headers)
    1167
    1168
    1169 -  def _id_to_header(self, id_):
    1170      """Convert an id to a Content-ID header value.
    1171
    1172      Args:
    1173        id_: string, identifier of individual request.
    1174
    1175      Returns:
    1176        A Content-ID header with the id_ encoded into it. A UUID is prepended to
    1177        the value because Content-ID headers are supposed to be universally
    1178        unique.
    1179      """
    1180      if self._base_id is None:
    1181        self._base_id = uuid.uuid4()
    1182
    1183      # NB: we intentionally leave whitespace between base/id and '+', so RFC2822
    1184      # line folding works properly on Python 3; see
    1185      # https://github.com/google/google-api-python-client/issues/164
    1186      return '<%s + %s>' % (self._base_id, quote(id_))
    1187
    1188 -  def _header_to_id(self, header):
    1189      """Convert a Content-ID header value to an id.
    1190
    1191      Presumes the Content-ID header conforms to the format that _id_to_header()
    1192      returns.
    1193
    1194      Args:
    1195        header: string, Content-ID header value.
    1196
    1197      Returns:
    1198        The extracted id value.
    1199
    1200      Raises:
    1201        BatchError if the header is not in the expected format.
    1202      """
    1203      if header[0] != '<' or header[-1] != '>':
    1204        raise BatchError("Invalid value for Content-ID: %s" % header)
    1205      if '+' not in header:
    1206        raise BatchError("Invalid value for Content-ID: %s" % header)
    1207      base, id_ = header[1:-1].split(' + ', 1)
    1208
    1209      return unquote(id_)
    1210
    1211 -  def _serialize_request(self, request):
    1212      """Convert an HttpRequest object into a string.
    1213
    1214      Args:
    1215        request: HttpRequest, the request to serialize.
    1216
    1217      Returns:
    1218        The request as a string in application/http format.
    1219      """
    1220      # Construct status line
    1221      parsed = urlparse(request.uri)
    1222      request_line = urlunparse(
    1223          ('', '', parsed.path, parsed.params, parsed.query, '')
    1224          )
    1225      status_line = request.method + ' ' + request_line + ' HTTP/1.1\n'
    1226      major, minor = request.headers.get('content-type', 'application/json').split('/')
    1227      msg = MIMENonMultipart(major, minor)
    1228      headers = request.headers.copy()
    1229
    1230      if request.http is not None:
    1231        credentials = _auth.get_credentials_from_http(request.http)
    1232        if credentials is not None:
    1233          _auth.apply_credentials(credentials, headers)
    1234
    1235      # MIMENonMultipart adds its own Content-Type header.
    1236      if 'content-type' in headers:
    1237        del headers['content-type']
    1238
    1239      for key, value in six.iteritems(headers):
    1240        msg[key] = value
    1241      msg['Host'] = parsed.netloc
    1242      msg.set_unixfrom(None)
    1243
    1244      if request.body is not None:
    1245        msg.set_payload(request.body)
    1246        msg['content-length'] = str(len(request.body))
    1247
    1248      # Serialize the mime message.
    1249      fp = StringIO()
    1250      # maxheaderlen=0 means don't line wrap headers.
    1251      g = Generator(fp, maxheaderlen=0)
    1252      g.flatten(msg, unixfrom=False)
    1253      body = fp.getvalue()
    1254
    1255      return status_line + body
    1256
    1257 -  def _deserialize_response(self, payload):
    1258      """Convert string into httplib2 response and content.
    1259
    1260      Args:
    1261        payload: string, headers and body as a string.
    1262
    1263      Returns:
    1264        A pair (resp, content), such as would be returned from httplib2.request.
    1265      """
    1266      # Strip off the status line
    1267      status_line, payload = payload.split('\n', 1)
    1268      protocol, status, reason = status_line.split(' ', 2)
    1269
    1270      # Parse the rest of the response
    1271      parser = FeedParser()
    1272      parser.feed(payload)
    1273      msg = parser.close()
    1274      msg['status'] = status
    1275
    1276      # Create httplib2.Response from the parsed headers.
    1277      resp = httplib2.Response(msg)
    1278      resp.reason = reason
    1279      resp.version = int(protocol.split('/', 1)[1].replace('.', ''))
    1280
    1281      content = payload.split('\r\n\r\n', 1)[1]
    1282
    1283      return resp, content
    1284
    1285 -  def _new_id(self):
    1286      """Create a new id.
    1287
    1288      Auto incrementing number that avoids conflicts with ids already used.
    1289
    1290      Returns:
    1291         string, a new unique id.
    1292      """
    1293      self._last_auto_id += 1
    1294      while str(self._last_auto_id) in self._requests:
    1295        self._last_auto_id += 1
    1296      return str(self._last_auto_id)
    1297
    1298    @util.positional(2)
    1299 -  def add(self, request, callback=None, request_id=None):
    1300      """Add a new request.
    1301
    1302      Every callback added will be paired with a unique id, the request_id. That
    1303      unique id will be passed back to the callback when the response comes back
    1304      from the server. The default behavior is to have the library generate it's
    1305      own unique id. If the caller passes in a request_id then they must ensure
    1306      uniqueness for each request_id, and if they are not an exception is
    1307      raised. Callers should either supply all request_ids or never supply a
    1308      request id, to avoid such an error.
    1309
    1310      Args:
    1311        request: HttpRequest, Request to add to the batch.
    1312        callback: callable, A callback to be called for this response, of the
    1313          form callback(id, response, exception). The first parameter is the
    1314          request id, and the second is the deserialized response object. The
    1315          third is an googleapiclient.errors.HttpError exception object if an HTTP error
    1316          occurred while processing the request, or None if no errors occurred.
    1317        request_id: string, A unique id for the request. The id will be passed
    1318          to the callback with the response.
    1319
    1320      Returns:
    1321        None
    1322
    1323      Raises:
    1324        BatchError if a media request is added to a batch.
    1325        KeyError is the request_id is not unique.
    1326      """
    1327
    1328      if len(self._order) >= MAX_BATCH_LIMIT:
    1329        raise BatchError("Exceeded the maximum calls(%d) in a single batch request."
    1330                         % MAX_BATCH_LIMIT)
    1331      if request_id is None:
    1332        request_id = self._new_id()
    1333      if request.resumable is not None:
    1334        raise BatchError("Media requests cannot be used in a batch request.")
    1335      if request_id in self._requests:
    1336        raise KeyError("A request with this ID already exists: %s" % request_id)
    1337      self._requests[request_id] = request
    1338      self._callbacks[request_id] = callback
    1339      self._order.append(request_id)
    1340
    1341 -  def _execute(self, http, order, requests):
    1342      """Serialize batch request, send to server, process response.
    1343
    1344      Args:
    1345        http: httplib2.Http, an http object to be used to make the request with.
    1346        order: list, list of request ids in the order they were added to the
    1347          batch.
    1348        request: list, list of request objects to send.
    1349
    1350      Raises:
    1351        httplib2.HttpLib2Error if a transport error has occured.
    1352        googleapiclient.errors.BatchError if the response is the wrong format.
    1353      """
    1354      message = MIMEMultipart('mixed')
    1355      # Message should not write out it's own headers.
    1356      setattr(message, '_write_headers', lambda self: None)
    1357
    1358      # Add all the individual requests.
    1359      for request_id in order:
    1360        request = requests[request_id]
    1361
    1362        msg = MIMENonMultipart('application', 'http')
    1363        msg['Content-Transfer-Encoding'] = 'binary'
    1364        msg['Content-ID'] = self._id_to_header(request_id)
    1365
    1366        body = self._serialize_request(request)
    1367        msg.set_payload(body)
    1368        message.attach(msg)
    1369
    1370      # encode the body: note that we can't use `as_string`, because
    1371      # it plays games with `From ` lines.
    1372      fp = StringIO()
    1373      g = Generator(fp, mangle_from_=False)
    1374      g.flatten(message, unixfrom=False)
    1375      body = fp.getvalue()
    1376
    1377      headers = {}
    1378      headers['content-type'] = ('multipart/mixed; '
    1379                                 'boundary="%s"') % message.get_boundary()
    1380
    1381      resp, content = http.request(self._batch_uri, method='POST', body=body,
    1382                                   headers=headers)
    1383
    1384      if resp.status >= 300:
    1385        raise HttpError(resp, content, uri=self._batch_uri)
    1386
    1387      # Prepend with a content-type header so FeedParser can handle it.
    1388      header = 'content-type: %s\r\n\r\n' % resp['content-type']
    1389      # PY3's FeedParser only accepts unicode. So we should decode content
    1390      # here, and encode each payload again.
    1391      if six.PY3:
    1392        content = content.decode('utf-8')
    1393      for_parser = header + content
    1394
    1395      parser = FeedParser()
    1396      parser.feed(for_parser)
    1397      mime_response = parser.close()
    1398
    1399      if not mime_response.is_multipart():
    1400        raise BatchError("Response not in multipart/mixed format.", resp=resp,
    1401                         content=content)
    1402
    1403      for part in mime_response.get_payload():
    1404        request_id = self._header_to_id(part['Content-ID'])
    1405        response, content = self._deserialize_response(part.get_payload())
    1406        # We encode content here to emulate normal http response.
    1407        if isinstance(content, six.text_type):
    1408          content = content.encode('utf-8')
    1409        self._responses[request_id] = (response, content)
    1410
    1411    @util.positional(1)
    1412 -  def execute(self, http=None):
    1413      """Execute all the requests as a single batched HTTP request.
    1414
    1415      Args:
    1416        http: httplib2.Http, an http object to be used in place of the one the
    1417          HttpRequest request object was constructed with. If one isn't supplied
    1418          then use a http object from the requests in this batch.
    1419
    1420      Returns:
    1421        None
    1422
    1423      Raises:
    1424        httplib2.HttpLib2Error if a transport error has occured.
    1425        googleapiclient.errors.BatchError if the response is the wrong format.
    1426      """
    1427      # If we have no requests return
    1428      if len(self._order) == 0:
    1429        return None
    1430
    1431      # If http is not supplied use the first valid one given in the requests.
    1432      if http is None:
    1433        for request_id in self._order:
    1434          request = self._requests[request_id]
    1435          if request is not None:
    1436            http = request.http
    1437            break
    1438
    1439      if http is None:
    1440        raise ValueError("Missing a valid http object.")
    1441
    1442      # Special case for OAuth2Credentials-style objects which have not yet been
    1443      # refreshed with an initial access_token.
    1444      creds = _auth.get_credentials_from_http(http)
    1445      if creds is not None:
    1446        if not _auth.is_valid(creds):
    1447          LOGGER.info('Attempting refresh to obtain initial access_token')
    1448          _auth.refresh_credentials(creds)
    1449
    1450      self._execute(http, self._order, self._requests)
    1451
    1452      # Loop over all the requests and check for 401s. For each 401 request the
    1453      # credentials should be refreshed and then sent again in a separate batch.
    1454      redo_requests = {}
    1455      redo_order = []
    1456
    1457      for request_id in self._order:
    1458        resp, content = self._responses[request_id]
    1459        if resp['status'] == '401':
    1460          redo_order.append(request_id)
    1461          request = self._requests[request_id]
    1462          self._refresh_and_apply_credentials(request, http)
    1463          redo_requests[request_id] = request
    1464
    1465      if redo_requests:
    1466        self._execute(http, redo_order, redo_requests)
    1467
    1468      # Now process all callbacks that are erroring, and raise an exception for
    1469      # ones that return a non-2xx response? Or add extra parameter to callback
    1470      # that contains an HttpError?
    1471
    1472      for request_id in self._order:
    1473        resp, content = self._responses[request_id]
    1474
    1475        request = self._requests[request_id]
    1476        callback = self._callbacks[request_id]
    1477
    1478        response = None
    1479        exception = None
    1480        try:
    1481          if resp.status >= 300:
    1482            raise HttpError(resp, content, uri=request.uri)
    1483          response = request.postproc(resp, content)
    1484        except HttpError as e:
    1485          exception = e
    1486
    1487        if callback is not None:
    1488          callback(request_id, response, exception)
    1489        if self._callback is not None:
    1490          self._callback(request_id, response, exception)
    1491
    1492
    1493 -class HttpRequestMock(object):
    1494    """Mock of HttpRequest.
    1495
    1496    Do not construct directly, instead use RequestMockBuilder.
    1497    """
    1498
    1499 -  def __init__(self, resp, content, postproc):
    1500      """Constructor for HttpRequestMock
    1501
    1502      Args:
    1503        resp: httplib2.Response, the response to emulate coming from the request
    1504        content: string, the response body
    1505        postproc: callable, the post processing function usually supplied by
    1506                  the model class. See model.JsonModel.response() as an example.
    1507      """
    1508      self.resp = resp
    1509      self.content = content
    1510      self.postproc = postproc
    1511      if resp is None:
    1512        self.resp = httplib2.Response({'status': 200, 'reason': 'OK'})
    1513      if 'reason' in self.resp:
    1514        self.resp.reason = self.resp['reason']
    1515
    1516 -  def execute(self, http=None):
    1517      """Execute the request.
    1518
    1519      Same behavior as HttpRequest.execute(), but the response is
    1520      mocked and not really from an HTTP request/response.
    1521      """
    1522      return self.postproc(self.resp, self.content)
    1523
    1524
    1525 -class RequestMockBuilder(object):
    1526    """A simple mock of HttpRequest
    1527
    1528      Pass in a dictionary to the constructor that maps request methodIds to
    1529      tuples of (httplib2.Response, content, opt_expected_body) that should be
    1530      returned when that method is called. None may also be passed in for the
    1531      httplib2.Response, in which case a 200 OK response will be generated.
    1532      If an opt_expected_body (str or dict) is provided, it will be compared to
    1533      the body and UnexpectedBodyError will be raised on inequality.
    1534
    1535      Example:
    1536        response = '{"data": {"id": "tag:google.c...'
    1537        requestBuilder = RequestMockBuilder(
    1538          {
    1539            'plus.activities.get': (None, response),
    1540          }
    1541        )
    1542        googleapiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder)
    1543
    1544      Methods that you do not supply a response for will return a
    1545      200 OK with an empty string as the response content or raise an excpetion
    1546      if check_unexpected is set to True. The methodId is taken from the rpcName
    1547      in the discovery document.
    1548
    1549      For more details see the project wiki.
    1550    """
    1551
    1552 -  def __init__(self, responses, check_unexpected=False):
    1553      """Constructor for RequestMockBuilder
    1554
    1555      The constructed object should be a callable object
    1556      that can replace the class HttpResponse.
    1557
    1558      responses - A dictionary that maps methodIds into tuples
    1559                  of (httplib2.Response, content). The methodId
    1560                  comes from the 'rpcName' field in the discovery
    1561                  document.
    1562      check_unexpected - A boolean setting whether or not UnexpectedMethodError
    1563                         should be raised on unsupplied method.
    1564      """
    1565      self.responses = responses
    1566      self.check_unexpected = check_unexpected
    1567
    1568 -  def __call__(self, http, postproc, uri, method='GET', body=None,
    1569                 headers=None, methodId=None, resumable=None):
    1570      """Implements the callable interface that discovery.build() expects
    1571      of requestBuilder, which is to build an object compatible with
    1572      HttpRequest.execute(). See that method for the description of the
    1573      parameters and the expected response.
    1574      """
    1575      if methodId in self.responses:
    1576        response = self.responses[methodId]
    1577        resp, content = response[:2]
    1578        if len(response) > 2:
    1579          # Test the body against the supplied expected_body.
    1580          expected_body = response[2]
    1581          if bool(expected_body) != bool(body):
    1582            # Not expecting a body and provided one
    1583            # or expecting a body and not provided one.
    1584            raise UnexpectedBodyError(expected_body, body)
    1585          if isinstance(expected_body, str):
    1586            expected_body = json.loads(expected_body)
    1587          body = json.loads(body)
    1588          if body != expected_body:
    1589            raise UnexpectedBodyError(expected_body, body)
    1590        return HttpRequestMock(resp, content, postproc)
    1591      elif self.check_unexpected:
    1592        raise UnexpectedMethodError(methodId=methodId)
    1593      else:
    1594        model = JsonModel(False)
    1595        return HttpRequestMock(None, '{}', model.response)
    1596
    1597
    1598 -class HttpMock(object):
    1599    """Mock of httplib2.Http"""
    1600
    1601 -  def __init__(self, filename=None, headers=None):
    1602      """
    1603      Args:
    1604        filename: string, absolute filename to read response from
    1605        headers: dict, header to return with response
    1606      """
    1607      if headers is None:
    1608        headers = {'status': '200'}
    1609      if filename:
    1610        f = open(filename, 'rb')
    1611        self.data = f.read()
    1612        f.close()
    1613      else:
    1614        self.data = None
    1615      self.response_headers = headers
    1616      self.headers = None
    1617      self.uri = None
    1618      self.method = None
    1619      self.body = None
    1620      self.headers = None
    1621
    1622
    1623 -  def request(self, uri,
    1624                method='GET',
    1625                body=None,
    1626                headers=None,
    1627                redirections=1,
    1628                connection_type=None):
    1629      self.uri = uri
    1630      self.method = method
    1631      self.body = body
    1632      self.headers = headers
    1633      return httplib2.Response(self.response_headers), self.data
    1634
    1635
    1636 -class HttpMockSequence(object):
    1637    """Mock of httplib2.Http
    1638
    1639    Mocks a sequence of calls to request returning different responses for each
    1640    call. Create an instance initialized with the desired response headers
    1641    and content and then use as if an httplib2.Http instance.
    1642
    1643      http = HttpMockSequence([
    1644        ({'status': '401'}, ''),
    1645        ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'),
    1646        ({'status': '200'}, 'echo_request_headers'),
    1647        ])
    1648      resp, content = http.request("http://examples.com")
    1649
    1650    There are special values you can pass in for content to trigger
    1651    behavours that are helpful in testing.
    1652
    1653    'echo_request_headers' means return the request headers in the response body
    1654    'echo_request_headers_as_json' means return the request headers in
    1655       the response body
    1656    'echo_request_body' means return the request body in the response body
    1657    'echo_request_uri' means return the request uri in the response body
    1658    """
    1659
    1660 -  def __init__(self, iterable):
    1661      """
    1662      Args:
    1663        iterable: iterable, a sequence of pairs of (headers, body)
    1664      """
    1665      self._iterable = iterable
    1666      self.follow_redirects = True
    1667
    1668 -  def request(self, uri,
    1669                method='GET',
    1670                body=None,
    1671                headers=None,
    1672                redirections=1,
    1673                connection_type=None):
    1674      resp, content = self._iterable.pop(0)
    1675      if content == 'echo_request_headers':
    1676        content = headers
    1677      elif content == 'echo_request_headers_as_json':
    1678        content = json.dumps(headers)
    1679      elif content == 'echo_request_body':
    1680        if hasattr(body, 'read'):
    1681          content = body.read()
    1682        else:
    1683          content = body
    1684      elif content == 'echo_request_uri':
    1685        content = uri
    1686      if isinstance(content, six.text_type):
    1687        content = content.encode('utf-8')
    1688      return httplib2.Response(resp), content
    1689
    1690
    1691 -def set_user_agent(http, user_agent):
    1692    """Set the user-agent on every request.
    1693
    1694    Args:
    1695       http - An instance of httplib2.Http
    1696           or something that acts like it.
    1697       user_agent: string, the value for the user-agent header.
    1698
    1699    Returns:
    1700       A modified instance of http that was passed in.
    1701
    1702    Example:
    1703
    1704      h = httplib2.Http()
    1705      h = set_user_agent(h, "my-app-name/6.0")
    1706
    1707    Most of the time the user-agent will be set doing auth, this is for the rare
    1708    cases where you are accessing an unauthenticated endpoint.
    1709    """
    1710    request_orig = http.request
    1711
    1712    # The closure that will replace 'httplib2.Http.request'.
    1713    def new_request(uri, method='GET', body=None, headers=None,
    1714                    redirections=httplib2.DEFAULT_MAX_REDIRECTS,
    1715                    connection_type=None):
    1716      """Modify the request headers to add the user-agent."""
    1717      if headers is None:
    1718        headers = {}
    1719      if 'user-agent' in headers:
    1720        headers['user-agent'] = user_agent + ' ' + headers['user-agent']
    1721      else:
    1722        headers['user-agent'] = user_agent
    1723      resp, content = request_orig(uri, method=method, body=body, headers=headers,
    1724                          redirections=redirections, connection_type=connection_type)
    1725      return resp, content
    1726
    1727    http.request = new_request
    1728    return http
    1729
    1730
    1731 -def tunnel_patch(http):
    1732    """Tunnel PATCH requests over POST.
    1733    Args:
    1734       http - An instance of httplib2.Http
    1735           or something that acts like it.
    1736
    1737    Returns:
    1738       A modified instance of http that was passed in.
    1739
    1740    Example:
    1741
    1742      h = httplib2.Http()
    1743      h = tunnel_patch(h, "my-app-name/6.0")
    1744
    1745    Useful if you are running on a platform that doesn't support PATCH.
    1746    Apply this last if you are using OAuth 1.0, as changing the method
    1747    will result in a different signature.
    1748    """
    1749    request_orig = http.request
    1750
    1751    # The closure that will replace 'httplib2.Http.request'.
    1752    def new_request(uri, method='GET', body=None, headers=None,
    1753                    redirections=httplib2.DEFAULT_MAX_REDIRECTS,
    1754                    connection_type=None):
    1755      """Modify the request headers to add the user-agent."""
    1756      if headers is None:
    1757        headers = {}
    1758      if method == 'PATCH':
    1759        if 'oauth_token' in headers.get('authorization', ''):
    1760          LOGGER.warning(
    1761              'OAuth 1.0 request made with Credentials after tunnel_patch.')
    1762        headers['x-http-method-override'] = "PATCH"
    1763        method = 'POST'
    1764      resp, content = request_orig(uri, method=method, body=body, headers=headers,
    1765                          redirections=redirections, connection_type=connection_type)
    1766      return resp, content
    1767
    1768    http.request = new_request
    1769    return http
    1770
    1771
    1772 -def build_http():
    1773    """Builds httplib2.Http object
    1774
    1775    Returns:
    1776    A httplib2.Http object, which is used to make http requests, and which has timeout set by default.
    1777    To override default timeout call
    1778
    1779      socket.setdefaulttimeout(timeout_in_sec)
    1780
    1781    before interacting with this method.
    1782    """
    1783    if socket.getdefaulttimeout() is not None:
    1784      http_timeout = socket.getdefaulttimeout()
    1785    else:
    1786      http_timeout = DEFAULT_HTTP_TIMEOUT_SEC
    1787    return httplib2.Http(timeout=http_timeout)
    1788