Skip to content

Provide example on how to throttle requests to live in rate limits #233

Open

Description

@nvkelso

Ian's done a bit of that for vector tiles in his Tilepacks work here.

def fetch_tile(format_args):
    sleep_time = 0.5 * random.uniform(1.0, 1.7)
    response_info = []
    while True:
        url = 'https://tile.mapzen.com/mapzen/vector/v1/{layer}/{zoom}/{x}/{y}.{fmt}?api_key={api_key}'.format(**format_args)
        try:
            start = time.time()

            resp = sess.get(url, timeout=(6.1, 30))

            data = resp.content
            finish = time.time()

            response_info.append({
                "url": url,
                "response status": resp.status_code,
                "server": resp.headers.get('Server'),
                "time to headers millis": int(resp.elapsed.total_seconds() * 1000),
                "time for content millis": int((finish - start) * 1000),
                "response length bytes": len(data),
            })

            resp.raise_for_status()
            return (format_args, response_info, data)
        except requests.exceptions.RequestException as e:
            if isinstance(e, requests.exceptions.HTTPError):
                if e.response.status_code == 404:
                    print("HTTP 404 -- {} while retrieving {}. Not trying again.".format(
                        e.response.status_code, e.response.text, url)
                    )
                    return (format_args, response_info, None)
                else:
                    print("HTTP error {} -- {} while retrieving {}, retrying after {:0.2f} sec".format(
                        e.response.status_code, e.response.text, url, sleep_time)
                    )
            else:
                print("{} while retrieving {}, retrying after {:0.2f} sec".format(
                    type(e), url, sleep_time)
                )
            time.sleep(sleep_time)
            sleep_time = min(sleep_time * 2.0, 30.0) * random.uniform(1.0, 1.7)
        except Exception as e:
            print("Ran into an unexpected exception: {}".format(e))
            traceback.print_tb()
            raise

Activity

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions