[game-data-packager] 02/02: urllib: use same Agent everywhere
Alexandre Detiste
detiste-guest at moszumanska.debian.org
Wed Sep 30 14:25:54 UTC 2015
This is an automated email from the git hooks/post-receive script.
detiste-guest pushed a commit to branch master
in repository game-data-packager.
commit 401df8c877cf4ccfe7016df5b6b6435e5d4f8728
Author: Alexandre Detiste <alexandre.detiste at gmail.com>
Date: Wed Sep 30 16:25:23 2015 +0200
urllib: use same Agent everywhere
---
game_data_packager/__init__.py | 5 +----
game_data_packager/steam.py | 3 ++-
game_data_packager/util.py | 5 +++++
tools/spider.py | 3 ++-
4 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/game_data_packager/__init__.py b/game_data_packager/__init__.py
index 10205a3..833851a 100644
--- a/game_data_packager/__init__.py
+++ b/game_data_packager/__init__.py
@@ -46,6 +46,7 @@ from .config import read_config
from .gog import GOG
from .paths import DATADIR, ETCDIR
from .util import (MEBIBYTE,
+ AGENT,
TemporaryUmask,
copy_with_substitutions,
mkdir_p,
@@ -58,10 +59,6 @@ from .util import (MEBIBYTE,
from .steam import get_steam_id, owned_steam_games
from .version import GAME_PACKAGE_VERSION
-AGENT = ('Debian Game-Data-Packager/%s (%s %s;'
- ' +http://wiki.debian.org/Games/GameDataPackager)' %
- (GAME_PACKAGE_VERSION, os.uname()[0], os.uname()[4]) )
-
logging.basicConfig()
logger = logging.getLogger('game-data-packager')
diff --git a/game_data_packager/steam.py b/game_data_packager/steam.py
index d39be1b..5e38b4d 100644
--- a/game_data_packager/steam.py
+++ b/game_data_packager/steam.py
@@ -19,6 +19,7 @@ import glob
import os
import xml.etree.ElementTree
import urllib.request
+from .util import AGENT
def parse_acf(path):
for manifest in glob.glob(path + '/*.acf'):
@@ -45,7 +46,7 @@ def parse_acf(path):
def owned_steam_games(steam_id):
url = "http://steamcommunity.com/profiles/" + steam_id + "/games?xml=1"
- html = urllib.request.urlopen(url)
+ html = urllib.request.urlopen(urllib.request.Request(url, headers={'User-Agent': AGENT}))
tree = xml.etree.ElementTree.ElementTree()
tree.parse(html)
games_xml = tree.getiterator('game')
diff --git a/game_data_packager/util.py b/game_data_packager/util.py
index 766ec65..bab69c5 100644
--- a/game_data_packager/util.py
+++ b/game_data_packager/util.py
@@ -21,10 +21,15 @@ import shutil
import stat
import subprocess
import sys
+from .version import GAME_PACKAGE_VERSION
KIBIBYTE = 1024
MEBIBYTE = KIBIBYTE * KIBIBYTE
+AGENT = ('Debian Game-Data-Packager/%s (%s %s;'
+ ' +http://wiki.debian.org/Games/GameDataPackager)' %
+ (GAME_PACKAGE_VERSION, os.uname()[0], os.uname()[4]) )
+
class TemporaryUmask(object):
"""Context manager to set the umask. Not thread-safe.
diff --git a/tools/spider.py b/tools/spider.py
index a966135..5eed6b6 100755
--- a/tools/spider.py
+++ b/tools/spider.py
@@ -24,6 +24,7 @@ import time
import urllib.request
from bs4 import BeautifulSoup
from game_data_packager import load_games
+from game_data_packager.util import AGENT
CSV = 'data/wikipedia.csv'
@@ -52,7 +53,7 @@ for shortname, game in load_games(None, game=todo).items():
print('processing %s ...' % shortname)
url = game.wikibase + game.wiki
- html = urllib.request.urlopen(url)
+ html = urllib.request.urlopen(urllib.request.Request(url,headers={'User-Agent': AGENT}))
soup = BeautifulSoup(html, 'lxml')
for tag in soup.find_all(href=is_wikipedia):
print(' ' + tag['href'])
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git
More information about the Pkg-games-commits
mailing list