Commit 819c104f authored by Michael Catanzaro's avatar Michael Catanzaro
Browse files

Sleep one second when receiving HTTP 429 error

One of our mirrors has started rate-limiting us. It seems this mirror is
selected for me 100% of the time, and tarball conversion fails 100% of
the time. Ouch. We can avoid this by sleeping for one second and
retrying when it occurs.

Note this is called on a thread, so it theoretically only affects the
thread that actaully received the 429 response. This means that other
threads are likely to hit the same error, but that's OK.

Fixes #23
parent 0d244793
......@@ -22,9 +22,10 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
# USA
import os
import re
import requests
import os
import time
from html.parser import HTMLParser
from posixpath import join as posixjoin # Handy for URLs
......@@ -38,6 +39,17 @@ class DownloadSite:
def find_tarball(self, modulename, max_version, wantchecksum):
raise NotImplementedError
def perform_request(location):
req = None
while True:
req = requests.get(location)
if req.status_code == 429:
# Too Many Requests: we hit a rate limit
time.sleep(1)
continue
req.raise_for_status()
return req
class Tarballs(DownloadSite):
def __init__(self, baseurl):
super().__init__(baseurl)
......@@ -50,8 +62,7 @@ class Tarballs(DownloadSite):
location = self.baseurl.format(module=modulename)
while True:
req = requests.get(location)
req.raise_for_status()
req = perform_request(location)
files = get_links(req.text)
# Check to see if we need to descend to a subdirectory
......@@ -94,8 +105,7 @@ class GNOME(DownloadSite):
def __init__(self, baseurl):
super().__init__(baseurl)
resp = requests.get(self.baseurl)
resp.raise_for_status()
resp = perform_request(self.baseurl)
moduleregex = re.compile('([^/]+)/')
......@@ -108,8 +118,7 @@ class GNOME(DownloadSite):
if modulename not in self.modules:
return None, None, None
resp = requests.get(posixjoin(self.baseurl, modulename, 'cache.json'))
resp.raise_for_status()
resp = perform_request(posixjoin(self.baseurl, modulename, 'cache.json'))
versions = resp.json()[1][modulename]
latest = get_latest_version(versions.keys(), max_version)
......@@ -128,8 +137,7 @@ class GNOME(DownloadSite):
checksum = None
if wantchecksum and 'sha256sum' in versions[latest]:
resp = requests.get(posixjoin(self.baseurl, modulename, versions[latest]['sha256sum']))
resp.raise_for_status()
resp = perform_request(posixjoin(self.baseurl, modulename, versions[latest]['sha256sum']))
basename = os.path.basename(tarball)
for l in resp.text.splitlines():
......
## CORE
core:adwaita-icon-theme:3.38.0:
core:at-spi2-atk:2.38.0:
core:at-spi2-core:2.38.0:
core:atk:2.36.0:
core:atkmm:2.28.1:
core:baobab:3.38.0:
core:cantarell-fonts:0.301:
core:cheese:3.38.0:
core:clutter:1.26.4:
core:clutter-gst:3.0.27:
core:clutter-gtk:1.8.4:
core:cogl:1.22.8:
core:dconf:0.38.0:
core:eog:3.38.1:
core:epiphany:3.38.2:
core:evince:3.38.0:
core:evolution-data-server:3.38.2:
core:file-roller:3.38.0:
core:folks:0.14.0:
core:gcab:1.4:
core:gcr:3.38.0:
core:gdk-pixbuf:2.42.2:
core:gdm:3.38.2.1:
core:gedit:3.38.1:
core:geocode-glib:3.26.2:
core:gjs:1.66.1:
core:glib:2.67.1:
core:glib-networking:2.66.0:
core:glibmm:2.64.5:
core:gmime:3.2.7:
core:gnome-autoar:0.2.4:
core:gnome-backgrounds:3.38.0:
core:gnome-bluetooth:3.34.3:
core:gnome-boxes:3.38.2:
core:gnome-calculator:3.38.2:
core:gnome-calendar:3.38.2:
core:gnome-characters:3.34.0:
core:gnome-clocks:3.38.0:
core:gnome-color-manager:3.36.0:
core:gnome-contacts:3.38.1:
core:gnome-control-center:3.38.2:
core:gnome-desktop:3.38.2:
core:gnome-disk-utility:3.38.1:
core:gnome-font-viewer:3.34.0:
core:gnome-getting-started-docs:3.38.0:
core:gnome-initial-setup:3.38.2:
core:gnome-keyring:3.36.0:
core:gnome-logs:3.36.0:
core:gnome-maps:3.38.2:
core:gnome-menus:3.36.0:
core:gnome-music:3.38.2:
core:gnome-online-accounts:3.38.0:
core:gnome-online-miners:3.34.0:
core:gnome-photos:3.38.0:
core:gnome-screenshot:3.38.0:
core:gnome-session:3.38.0:
core:gnome-settings-daemon:3.38.1:
core:gnome-shell:3.38.2:
core:gnome-shell-extensions:3.38.2:
core:gnome-software:3.38.0:
core:gnome-system-monitor:3.38.0:
core:gnome-terminal:3.38.1:
core:gnome-user-docs:3.38.2:
core:gnome-user-share:3.34.0:
core:gnome-video-effects:0.5.0:
core:gnome-weather:3.36.1:
core:gobject-introspection:1.66.1:
core:gom:0.4:
core:grilo:0.3.13:
core:grilo-plugins:0.3.12:
core:gsettings-desktop-schemas:3.38.0:
core:gsound:1.0.2:
core:gspell:1.9.1:
core:gssdp:1.2.3:
core:gtk:4.0.0:
core:gtk+:3.24.24:
core:gtk-doc:1.33.1:
core:gtk-vnc:1.0.0:
core:gtkmm:3.24.3:
core:gtksourceview:4.8.0:
core:gupnp:1.2.4:
core:gupnp-av:0.12.11:
core:gupnp-dlna:0.10.5:
core:gvfs:1.46.1:
core:json-glib:1.6.0:
core:libchamplain:0.12.20:
core:libdazzle:3.38.0:
core:libgdata:0.17.13:
core:libgee:0.20.3:
core:gfbgraph:0.2.4:
core:libgnomekbd:3.26.1:
core:libgsf:1.14.47:
core:libgtop:2.40.0:
core:libgweather:3.36.1:
core:libgxps:0.3.1:
core:libhandy:1.0.2:
core:libmediaart:1.9.4:
core:libnma:1.8.30:
core:libnotify:0.7.9:
core:libpeas:1.28.0:
core:rest:0.8.1:
core:librsvg:2.50.2:
core:libsecret:0.20.4:
core:libsigc++:2.10.6:
core:libsoup:2.72.0:
core:mm-common:1.0.2:
core:mutter:3.38.2:
core:nautilus:3.38.2:
core:orca:3.38.2:
core:pango:1.48.0:
core:pangomm:2.42.2:
core:phodav:2.5:
core:pyatspi:2.38.0:
core:pygobject:3.38.0:
core:rygel:0.40.0:
core:simple-scan:3.38.2:
core:sushi:3.38.0:
core:totem:3.38.0:
core:totem-pl-parser:3.26.5:
core:tracker:3.0.2:
core:tracker:2.3.6:
core:tracker-miners:3.0.3:
core:tracker-miners:2.3.5:
core:vala:0.50.2:
core:vte:0.62.1:
core:yelp:3.38.2:
core:yelp-tools:3.38.0:
core:yelp-xsl:3.38.2:
core:zenity:3.32.0:
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment