Use requests-cache for adding etag/last-modified based caching

This doesn't speed things up usually, since we still make the same amount
of requests, but it doesn't count against the rate-limit in case there
is a cache hit. Also there is a smaller chance of things going wrong,
since we don't transfer any payload.

The cache is store in a .autobuild_cache directory using a sqlite DB.
This commit is contained in:
Christoph Reiter 2022-04-30 16:20:36 +02:00
parent 258256e739
commit 8870b3a342
5 changed files with 205 additions and 39 deletions

3
.gitignore vendored
View File

@ -1,3 +1,4 @@
*.pyc
.vscode/
.mypy_cache/
.mypy_cache/
.autobuild_cache/

View File

@ -328,22 +328,23 @@ def get_asset_mtime_ns(asset: GitReleaseAsset) -> int:
def download_asset(asset: GitReleaseAsset, target_path: str) -> None:
assert asset_is_complete(asset)
session = get_requests_session()
with session.get(asset.browser_download_url, stream=True, timeout=REQUESTS_TIMEOUT) as r:
r.raise_for_status()
fd, temppath = tempfile.mkstemp()
try:
os.chmod(temppath, 0o644)
with os.fdopen(fd, "wb") as h:
for chunk in r.iter_content(4096):
h.write(chunk)
mtime_ns = get_asset_mtime_ns(asset)
os.utime(temppath, ns=(mtime_ns, mtime_ns))
shutil.move(temppath, target_path)
finally:
with requests_cache_disabled():
with session.get(asset.browser_download_url, stream=True, timeout=REQUESTS_TIMEOUT) as r:
r.raise_for_status()
fd, temppath = tempfile.mkstemp()
try:
os.remove(temppath)
except OSError:
pass
os.chmod(temppath, 0o644)
with os.fdopen(fd, "wb") as h:
for chunk in r.iter_content(4096):
h.write(chunk)
mtime_ns = get_asset_mtime_ns(asset)
os.utime(temppath, ns=(mtime_ns, mtime_ns))
shutil.move(temppath, target_path)
finally:
try:
os.remove(temppath)
except OSError:
pass
def download_text_asset(asset: GitReleaseAsset) -> str:
@ -1596,7 +1597,39 @@ def clean_environ(environ: Dict[str, str]) -> Dict[str, str]:
return new_env
def install_requests_cache() -> None:
# This adds basic etag based caching, to avoid hitting API rate limiting
import requests_cache
from requests_cache.backends.sqlite import SQLiteCache
# Monkey patch globally, so pygithub uses it as well.
# Only do re-validation with etag/date etc and ignore the cache-control headers that
# github sends by default with 60 seconds. This is only possible with requests_cache 0.10+
cache_dir = os.path.join(SCRIPT_DIR, '.autobuild_cache')
os.makedirs(cache_dir, exist_ok=True)
requests_cache.install_cache(
cache_control=False,
expire_after=0,
backend=SQLiteCache(os.path.join(cache_dir, 'http_cache.sqlite')))
# How to limit the cache size is an open question, at least to me:
# https://github.com/reclosedev/requests-cache/issues/620
# so do it the simple/stupid way
cache = requests_cache.get_cache()
assert cache is not None
if cache.response_count() > 200:
cache.clear()
def requests_cache_disabled() -> Any:
import requests_cache
return requests_cache.disabled()
def main(argv: List[str]) -> None:
install_requests_cache()
parser = argparse.ArgumentParser(description="Build packages", allow_abbrev=False)
parser.set_defaults(func=lambda *x: parser.print_help())
parser.add_argument(

162
poetry.lock generated
View File

@ -1,3 +1,29 @@
[[package]]
name = "attrs"
version = "21.4.0"
description = "Classes Without Boilerplate"
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
[package.extras]
dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"]
docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"]
tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"]
tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"]
[[package]]
name = "cattrs"
version = "22.1.0"
description = "Composable complex class support for attrs and dataclasses."
category = "main"
optional = false
python-versions = ">=3.7,<4.0"
[package.dependencies]
attrs = ">=20"
exceptiongroup = {version = "*", markers = "python_version <= \"3.10\""}
[[package]]
name = "certifi"
version = "2021.10.8"
@ -42,6 +68,17 @@ wrapt = ">=1.10,<2"
[package.extras]
dev = ["tox", "bump2version (<1)", "sphinx (<2)", "importlib-metadata (<3)", "importlib-resources (<4)", "configparser (<5)", "sphinxcontrib-websupport (<2)", "zipp (<2)", "PyTest (<5)", "PyTest-Cov (<2.6)", "pytest", "pytest-cov"]
[[package]]
name = "exceptiongroup"
version = "1.0.0rc4"
description = "Backport of PEP 654 (exception groups)"
category = "main"
optional = false
python-versions = ">=3.7"
[package.extras]
test = ["pytest (>=6)"]
[[package]]
name = "flake8"
version = "4.0.1"
@ -96,6 +133,18 @@ category = "dev"
optional = false
python-versions = "*"
[[package]]
name = "platformdirs"
version = "2.5.2"
description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
category = "main"
optional = false
python-versions = ">=3.7"
[package.extras]
docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)", "sphinx (>=4)"]
test = ["appdirs (==1.4.4)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)", "pytest (>=6)"]
[[package]]
name = "pycodestyle"
version = "2.8.0"
@ -184,6 +233,42 @@ urllib3 = ">=1.21.1,<1.27"
socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"]
use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"]
[[package]]
name = "requests-cache"
version = "0.10.0.dev4"
description = "A persistent cache for the requests library"
category = "main"
optional = false
python-versions = ">=3.7,<4.0"
[package.dependencies]
attrs = ">=21.2"
cattrs = ">=22.1"
exceptiongroup = {version = ">=1.0.0-rc.3,<2.0.0", markers = "python_full_version >= \"3.10.0\" and python_full_version < \"4.0.0\""}
platformdirs = ">=2.5,<3.0"
requests = ">=2.22,<3.0"
url-normalize = ">=1.4,<2.0"
urllib3 = ">=1.25.5"
[package.extras]
dynamodb = ["boto3 (>=1.15,<2.0)", "botocore (>=1.18,<2.0)"]
all = ["boto3 (>=1.15,<2.0)", "botocore (>=1.18,<2.0)", "pymongo (>=3)", "redis (>=3)", "itsdangerous (>=2.0,<3.0)", "pyyaml (>=5.4)", "ujson (>=4.0)"]
mongodb = ["pymongo (>=3)"]
redis = ["redis (>=3)"]
bson = ["bson (>=0.5)"]
security = ["itsdangerous (>=2.0,<3.0)"]
yaml = ["pyyaml (>=5.4)"]
json = ["ujson (>=4.0)"]
docs = ["furo (>=2022.4,<2023.0)", "linkify-it-py (>=1.0,<2.0)", "myst-parser (>=0.17)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.18,<2.0)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-inline-tabs (>=2022.1.2b11)", "sphinx-notfound-page (>=0.8)", "sphinx-panels (>=0.6,<0.7)", "sphinxcontrib-apidoc (>=0.3,<0.4)"]
[[package]]
name = "six"
version = "1.16.0"
description = "Python 2 and 3 compatibility utilities"
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
[[package]]
name = "tabulate"
version = "0.8.9"
@ -205,7 +290,7 @@ python-versions = ">=3.7"
[[package]]
name = "types-requests"
version = "2.27.11"
version = "2.27.25"
description = "Typing stubs for requests"
category = "dev"
optional = false
@ -216,7 +301,7 @@ types-urllib3 = "<1.27"
[[package]]
name = "types-tabulate"
version = "0.8.5"
version = "0.8.8"
description = "Typing stubs for tabulate"
category = "dev"
optional = false
@ -224,7 +309,7 @@ python-versions = "*"
[[package]]
name = "types-urllib3"
version = "1.26.10"
version = "1.26.14"
description = "Typing stubs for urllib3"
category = "dev"
optional = false
@ -232,22 +317,33 @@ python-versions = "*"
[[package]]
name = "typing-extensions"
version = "4.1.1"
description = "Backported and Experimental Type Hints for Python 3.6+"
version = "4.2.0"
description = "Backported and Experimental Type Hints for Python 3.7+"
category = "dev"
optional = false
python-versions = ">=3.6"
python-versions = ">=3.7"
[[package]]
name = "url-normalize"
version = "1.4.3"
description = "URL normalization for Python"
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
[package.dependencies]
six = "*"
[[package]]
name = "urllib3"
version = "1.26.8"
version = "1.26.9"
description = "HTTP library with thread-safe connection pooling, file post, and more."
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
[package.extras]
brotli = ["brotlipy (>=0.6.0)"]
brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"]
secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"]
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
@ -262,9 +358,17 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7"
[metadata]
lock-version = "1.1"
python-versions = "^3.8"
content-hash = "5a7d7346e3632a0a938248c5ab54d4116162cdff2c770a99f7a187a8309ed338"
content-hash = "7cb68bc4c6f3de750d6994e9145ba217dd8ae79a614c1b7cbea803a96980ab2e"
[metadata.files]
attrs = [
{file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"},
{file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"},
]
cattrs = [
{file = "cattrs-22.1.0-py3-none-any.whl", hash = "sha256:d55c477b4672f93606e992049f15d526dc7867e6c756cd6256d4af92e2b1e364"},
{file = "cattrs-22.1.0.tar.gz", hash = "sha256:94b67b64cf92c994f8784c40c082177dc916e0489a73a9a36b24eb18a9db40c6"},
]
certifi = [
{file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"},
{file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"},
@ -329,6 +433,10 @@ deprecated = [
{file = "Deprecated-1.2.13-py2.py3-none-any.whl", hash = "sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d"},
{file = "Deprecated-1.2.13.tar.gz", hash = "sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d"},
]
exceptiongroup = [
{file = "exceptiongroup-1.0.0rc4-py3-none-any.whl", hash = "sha256:21739e657be44a7298757e21227726ed4553e62b3ccb0be9ca5c51a2707d0f52"},
{file = "exceptiongroup-1.0.0rc4.tar.gz", hash = "sha256:b4d7649f94c0317251deb26799c549520072e693d22007e97ceca000421eb2c9"},
]
flake8 = [
{file = "flake8-4.0.1-py2.py3-none-any.whl", hash = "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d"},
{file = "flake8-4.0.1.tar.gz", hash = "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"},
@ -367,6 +475,10 @@ mypy-extensions = [
{file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"},
{file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"},
]
platformdirs = [
{file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"},
{file = "platformdirs-2.5.2.tar.gz", hash = "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"},
]
pycodestyle = [
{file = "pycodestyle-2.8.0-py2.py3-none-any.whl", hash = "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20"},
{file = "pycodestyle-2.8.0.tar.gz", hash = "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"},
@ -403,6 +515,14 @@ requests = [
{file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"},
{file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"},
]
requests-cache = [
{file = "requests-cache-0.10.0dev4.tar.gz", hash = "sha256:43414dd00263f37f0a4955fe8f5ab9b53b10ba893354ee0cf3370510d822bd5d"},
{file = "requests_cache-0.10.0dev4-py3-none-any.whl", hash = "sha256:e43e759f7ea40d0b7e9eaf1ad7daf5d0020e22b5521678db7410b88ffd1aebf2"},
]
six = [
{file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
{file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
]
tabulate = [
{file = "tabulate-0.8.9-py3-none-any.whl", hash = "sha256:d7c013fe7abbc5e491394e10fa845f8f32fe54f8dc60c6622c6cf482d25d47e4"},
{file = "tabulate-0.8.9.tar.gz", hash = "sha256:eb1d13f25760052e8931f2ef80aaf6045a6cceb47514db8beab24cded16f13a7"},
@ -412,24 +532,28 @@ tomli = [
{file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
]
types-requests = [
{file = "types-requests-2.27.11.tar.gz", hash = "sha256:6a7ed24b21780af4a5b5e24c310b2cd885fb612df5fd95584d03d87e5f2a195a"},
{file = "types_requests-2.27.11-py3-none-any.whl", hash = "sha256:506279bad570c7b4b19ac1f22e50146538befbe0c133b2cea66a9b04a533a859"},
{file = "types-requests-2.27.25.tar.gz", hash = "sha256:805ae7e38fd9d157153066dc4381cf585fd34dfa212f2fc1fece248c05aac571"},
{file = "types_requests-2.27.25-py3-none-any.whl", hash = "sha256:2444905c89731dbcb6bbcd6d873a04252445df7623917c640e463b2b28d2a708"},
]
types-tabulate = [
{file = "types-tabulate-0.8.5.tar.gz", hash = "sha256:03f283bf384ea121b7b6a58afb38f4def97f30704fca13a0da686936b0f392ac"},
{file = "types_tabulate-0.8.5-py3-none-any.whl", hash = "sha256:a7289b809220dfbb6d8816a8dcc25bdf82b91b47288225afb2c3f296e0417c7f"},
{file = "types-tabulate-0.8.8.tar.gz", hash = "sha256:0b319c7e10dd561f7c15f56dec347c3193fd865f4ebe5ff1ecb020eafa793bc3"},
{file = "types_tabulate-0.8.8-py3-none-any.whl", hash = "sha256:628912bf968b449fc8603f5a92a45739d464da38a6eb25d9e0b78a353a6e584c"},
]
types-urllib3 = [
{file = "types-urllib3-1.26.10.tar.gz", hash = "sha256:a26898f530e6c3f43f25b907f2b884486868ffd56a9faa94cbf9b3eb6e165d6a"},
{file = "types_urllib3-1.26.10-py3-none-any.whl", hash = "sha256:d755278d5ecd7a7a6479a190e54230f241f1a99c19b81518b756b19dc69e518c"},
{file = "types-urllib3-1.26.14.tar.gz", hash = "sha256:2a2578e4b36341ccd240b00fccda9826988ff0589a44ba4a664bbd69ef348d27"},
{file = "types_urllib3-1.26.14-py3-none-any.whl", hash = "sha256:5d2388aa76395b1e3999ff789ea5b3283677dad8e9bcf3d9117ba19271fd35d9"},
]
typing-extensions = [
{file = "typing_extensions-4.1.1-py3-none-any.whl", hash = "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"},
{file = "typing_extensions-4.1.1.tar.gz", hash = "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42"},
{file = "typing_extensions-4.2.0-py3-none-any.whl", hash = "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708"},
{file = "typing_extensions-4.2.0.tar.gz", hash = "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"},
]
url-normalize = [
{file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"},
{file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"},
]
urllib3 = [
{file = "urllib3-1.26.8-py2.py3-none-any.whl", hash = "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed"},
{file = "urllib3-1.26.8.tar.gz", hash = "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"},
{file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"},
{file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"},
]
wrapt = [
{file = "wrapt-1.14.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:5a9a1889cc01ed2ed5f34574c90745fab1dd06ec2eee663e8ebeefe363e8efd7"},

View File

@ -9,6 +9,7 @@ python = "^3.8"
PyGithub = "^1.54.1"
tabulate = "^0.8.7"
requests = "^2.25.1"
requests-cache = {version = "^0.10.0.dev4 "}
[tool.poetry.dev-dependencies]
mypy = "^0.931"

View File

@ -1,13 +1,20 @@
certifi==2021.10.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
attrs==21.4.0; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.5.0"
cattrs==22.1.0; python_version >= "3.7" and python_version < "4.0"
certifi==2021.10.8; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.6.0" and python_version >= "3.7" and python_version < "4.0"
cffi==1.15.0; python_version >= "3.6"
charset-normalizer==2.0.12; python_full_version >= "3.6.0" and python_version >= "3.6"
charset-normalizer==2.0.12; python_full_version >= "3.6.0" and python_version >= "3.7" and python_version < "4.0"
deprecated==1.2.13; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
idna==3.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
exceptiongroup==1.0.0rc4; python_full_version >= "3.10.0" and python_full_version < "4.0.0" and python_version >= "3.7" and python_version <= "3.10"
idna==3.3; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.6.0" and python_version >= "3.7" and python_version < "4.0"
platformdirs==2.5.2; python_version >= "3.7" and python_version < "4.0"
pycparser==2.21; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
pygithub==1.55; python_version >= "3.6"
pyjwt==2.3.0; python_version >= "3.6"
pynacl==1.5.0; python_version >= "3.6"
requests-cache==0.10.0.dev4; python_version >= "3.7" and python_version < "4.0"
requests==2.27.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
six==1.16.0; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.0"
tabulate==0.8.9
urllib3==1.26.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6"
url-normalize==1.4.3; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.0"
urllib3==1.26.9; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.7"
wrapt==1.14.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"