[geneagrapher] 221/226: Change grabber classes to return a dictionary.
Doug Torrance
dtorrance-guest at moszumanska.debian.org
Sat Jul 11 17:11:15 UTC 2015
This is an automated email from the git hooks/post-receive script.
dtorrance-guest pushed a commit to branch master
in repository geneagrapher.
commit ed5f80f3442a873caddf1980b6865537d98eed2c
Author: David Alber <alber.david at gmail.com>
Date: Sat Jan 7 23:39:13 2012 -0800
Change grabber classes to return a dictionary.
Grabber classes currently return a tuple of values with at least
name, institution, year, advisors, and descendants. Some grabbers
may have additional information to return, such as a string to
print when running Geneagrapher in verbose mode. Using a dictionary
as the return allows for more flexibility.
---
src/geneagrapher/cache_grabber.py | 25 ++++----
src/geneagrapher/geneagrapher.py | 21 +++----
src/geneagrapher/grabber.py | 13 +++--
tests/geneagrapher/test_cache_grabber.py | 92 ++++++++++++++++--------------
tests/geneagrapher/test_grabber_methods.py | 51 +++++++++--------
5 files changed, 103 insertions(+), 99 deletions(-)
diff --git a/src/geneagrapher/cache_grabber.py b/src/geneagrapher/cache_grabber.py
index 612f73d..523bfcd 100644
--- a/src/geneagrapher/cache_grabber.py
+++ b/src/geneagrapher/cache_grabber.py
@@ -31,16 +31,14 @@ class CacheGrabber():
id."""
id_str = str(id)
if self.is_cached(id_str):
- d = self.cache[id_str]
- return [d['name'], d['institution'], d['year'], d['advisors'],
- d['descendants'], 'cache hit']
+ record = self.cache[id_str]
+ record['message'] = 'cache hit'
else:
- [name, institution, year, advisors,
- descendants] = self.grabber.get_record(id)
- self.load_into_cache(id, name, institution, year, advisors,
- descendants)
- return [name, institution, year, advisors, descendants,
- 'cache miss']
+ record = self.grabber.get_record(id)
+ self.load_into_cache(id, record)
+ record['message'] = 'cache miss'
+ del(record['timestamp'])
+ return record
def is_cached(self, id):
"""Return True if an item with the given id is in the cache and has
@@ -48,13 +46,10 @@ class CacheGrabber():
return str(id) in self.cache and \
not self.is_expired(self.cache[str(id)])
- def load_into_cache(self, id, name, institution, year, advisors,
- descendants):
+ def load_into_cache(self, id, record):
"""Insert a new record into the cache.
If the record already exists, its values are replaced with the values
provided as input to this method."""
- d = {'name': name, 'institution': institution, 'year': year,
- 'advisors': advisors, 'descendants': descendants,
- 'timestamp': time()}
- self.cache[str(id)] = d
+ record['timestamp'] = time()
+ self.cache[str(id)] = record
diff --git a/src/geneagrapher/geneagrapher.py b/src/geneagrapher/geneagrapher.py
index fa7dea2..e58c108 100644
--- a/src/geneagrapher/geneagrapher.py
+++ b/src/geneagrapher/geneagrapher.py
@@ -82,21 +82,18 @@ geneacache]',
if self.verbose:
sys.stdout.write('Grabbing record #{}...'.format(id))
record = grabber.get_record(id)
- if len(record) == 5:
- [name, institution, year, advisors,
- descendants] = record
- grab_msg = ''
- elif len(record) == 6:
- [name, institution, year, advisors,
- descendants, grab_msg] = record
if self.verbose:
- print grab_msg
- self.graph.add_node(name, institution, year, id, advisors,
- descendants, is_seed)
+ if 'message' in record:
+ print record['message']
+ else:
+ print
+ self.graph.add_node(record['name'], record['institution'],
+ record['year'], id, record['advisors'],
+ record['descendants'], is_seed)
if self.get_ancestors and 'ancestor_queue' in kwargs:
- kwargs['ancestor_queue'].extend(advisors)
+ kwargs['ancestor_queue'].extend(record['advisors'])
if self.get_descendants and 'descendant_queue' in kwargs:
- kwargs['descendant_queue'].extend(descendants)
+ kwargs['descendant_queue'].extend(record['descendants'])
def build_graph_complete(self, record_grabber=Grabber, **kwargs):
"""
diff --git a/src/geneagrapher/grabber.py b/src/geneagrapher/grabber.py
index f38eee3..d3b79fb 100644
--- a/src/geneagrapher/grabber.py
+++ b/src/geneagrapher/grabber.py
@@ -39,12 +39,13 @@ def get_record_from_tree(soup, id):
msg = "Invalid id {}".format(id)
raise ValueError(msg)
- name = get_name(soup)
- institution = get_institution(soup)
- year = get_year(soup)
- advisors = get_advisors(soup)
- descendants = get_descendants(soup)
- return [name, institution, year, advisors, descendants]
+ record = {}
+ record['name'] = get_name(soup)
+ record['institution'] = get_institution(soup)
+ record['year'] = get_year(soup)
+ record['advisors'] = get_advisors(soup)
+ record['descendants'] = get_descendants(soup)
+ return record
def has_record(soup):
diff --git a/tests/geneagrapher/test_cache_grabber.py b/tests/geneagrapher/test_cache_grabber.py
index f4ce55a..df635cf 100644
--- a/tests/geneagrapher/test_cache_grabber.py
+++ b/tests/geneagrapher/test_cache_grabber.py
@@ -103,32 +103,35 @@ class TestCacheGrabberMethods(unittest.TestCase):
def test_get_record(self):
"""Test the get_record method for a good id."""
with CacheGrabber(record_grabber=LocalDataGrabber) as cache:
- [name, institution, year, advisors,
- descendants, msg] = cache.get_record(18231)
- self.assertEqual(name, u"Carl Friedrich Gau\xdf")
- self.assertEqual(institution, u"Universit\xe4t Helmstedt")
- self.assertEqual(year, 1799)
- self.assertEqual(advisors, set([18230]))
- self.assertEqual(descendants, set([18603, 18233, 62547, 29642,
- 55175, 29458, 19953, 18232,
- 151876]))
- self.assertEqual(msg, u"cache miss")
+ record = cache.get_record(18231)
+ self.assertEqual(len(record), 6)
+ self.assertEqual(record['name'], u"Carl Friedrich Gau\xdf")
+ self.assertEqual(record['institution'],
+ u"Universit\xe4t Helmstedt")
+ self.assertEqual(record['year'], 1799)
+ self.assertEqual(record['advisors'], set([18230]))
+ self.assertEqual(record['descendants'], set([18603, 18233, 62547,
+ 29642, 55175, 29458,
+ 19953, 18232,
+ 151876]))
+ self.assertEqual(record['message'], u"cache miss")
self.assertEqual(len(cache.cache), 1)
# Make the request again and verify the cached version is returned.
d = cache.cache['18231']
d['institution'] = u'Rigged for test'
cache.cache['18231'] = d
- [name, institution, year, advisors,
- descendants, msg] = cache.get_record(18231)
- self.assertEqual(name, u"Carl Friedrich Gau\xdf")
- self.assertEqual(institution, u"Rigged for test")
- self.assertEqual(year, 1799)
- self.assertEqual(advisors, set([18230]))
- self.assertEqual(descendants, set([18603, 18233, 62547, 29642,
- 55175, 29458, 19953, 18232,
- 151876]))
- self.assertEqual(msg, u"cache hit")
+ record = cache.get_record(18231)
+ self.assertEqual(len(record), 6)
+ self.assertEqual(record['name'], u"Carl Friedrich Gau\xdf")
+ self.assertEqual(record['institution'], u"Rigged for test")
+ self.assertEqual(record['year'], 1799)
+ self.assertEqual(record['advisors'], set([18230]))
+ self.assertEqual(record['descendants'], set([18603, 18233, 62547,
+ 29642, 55175, 29458,
+ 19953, 18232,
+ 151876]))
+ self.assertEqual(record['message'], u"cache hit")
self.assertEqual(len(cache.cache), 1)
with CacheGrabber(record_grabber=LocalDataGrabber) as cache:
@@ -138,32 +141,35 @@ class TestCacheGrabberMethods(unittest.TestCase):
d = cache.cache['18231']
d['institution'] = u'Rigged for test'
cache.cache['18231'] = d
- [name, institution, year, advisors,
- descendants, msg] = cache.get_record(18231)
- self.assertEqual(name, u"Carl Friedrich Gau\xdf")
- self.assertEqual(institution, u"Rigged for test")
- self.assertEqual(year, 1799)
- self.assertEqual(advisors, set([18230]))
- self.assertEqual(descendants, set([18603, 18233, 62547, 29642,
- 55175, 29458, 19953, 18232,
- 151876]))
- self.assertEqual(msg, u"cache hit")
+ record = cache.get_record(18231)
+ self.assertEqual(len(record), 6)
+ self.assertEqual(record['name'], u"Carl Friedrich Gau\xdf")
+ self.assertEqual(record['institution'], u"Rigged for test")
+ self.assertEqual(record['year'], 1799)
+ self.assertEqual(record['advisors'], set([18230]))
+ self.assertEqual(record['descendants'], set([18603, 18233, 62547,
+ 29642, 55175, 29458,
+ 19953, 18232,
+ 151876]))
+ self.assertEqual(record['message'], u"cache hit")
self.assertEqual(len(cache.cache), 1)
# Make another request, this time with the cached entry expired,
# and verify a new version is retrieved.
d['timestamp'] = time() - cache.expiration_interval - 1
cache.cache['18231'] = d
- [name, institution, year, advisors,
- descendants, msg] = cache.get_record(18231)
- self.assertEqual(name, u"Carl Friedrich Gau\xdf")
- self.assertEqual(institution, u"Universit\xe4t Helmstedt")
- self.assertEqual(year, 1799)
- self.assertEqual(advisors, set([18230]))
- self.assertEqual(descendants, set([18603, 18233, 62547, 29642,
- 55175, 29458, 19953, 18232,
- 151876]))
- self.assertEqual(msg, u"cache miss")
+ record = cache.get_record(18231)
+ self.assertEqual(len(record), 6)
+ self.assertEqual(record['name'], u"Carl Friedrich Gau\xdf")
+ self.assertEqual(record['institution'],
+ u"Universit\xe4t Helmstedt")
+ self.assertEqual(record['year'], 1799)
+ self.assertEqual(record['advisors'], set([18230]))
+ self.assertEqual(record['descendants'], set([18603, 18233, 62547,
+ 29642, 55175, 29458,
+ 19953, 18232,
+ 151876]))
+ self.assertEqual(record['message'], u"cache miss")
self.assertEqual(len(cache.cache), 1)
def test_is_in_cache(self):
@@ -184,8 +190,10 @@ class TestCacheGrabberMethods(unittest.TestCase):
"""Test the load_into_cache method."""
with CacheGrabber(record_grabber=LocalDataGrabber) as cache:
self.assertEqual(len(cache.cache), 0)
- cache.load_into_cache(self.id, self.name, self.institution,
- self.year, self.advisors, self.descendants)
+ new_record = {'name': self.name, 'institution': self.institution,
+ 'year': self.year, 'advisors': self.advisors,
+ 'descendants': self.descendants}
+ cache.load_into_cache(self.id, new_record)
self.assertEqual(len(cache.cache), 1)
record = cache.cache[str(self.id)]
self.assertEqual(record['name'], self.name)
diff --git a/tests/geneagrapher/test_grabber_methods.py b/tests/geneagrapher/test_grabber_methods.py
index 4b3c04a..a711738 100644
--- a/tests/geneagrapher/test_grabber_methods.py
+++ b/tests/geneagrapher/test_grabber_methods.py
@@ -31,14 +31,15 @@ class TestGrabberMethods(unittest.TestCase):
def test_get_record_all_fields(self):
"""Test the get_record() method for a record containing all fields."""
grabber = Grabber()
- [name, institution, year, advisors,
- descendents] = grabber.get_record(18231)
- self.assertEqual(name, u"Carl Friedrich Gau\xdf")
- self.assertEqual(institution, u"Universit\xe4t Helmstedt")
- self.assertEqual(year, 1799)
- self.assertEqual(advisors, set([18230]))
- self.assertEqual(descendents, set([18603, 18233, 62547, 29642, 55175,
- 29458, 19953, 18232, 151876]))
+ record = grabber.get_record(18231)
+ self.assertEqual(len(record), 5)
+ self.assertEqual(record['name'], u"Carl Friedrich Gau\xdf")
+ self.assertEqual(record['institution'], u"Universit\xe4t Helmstedt")
+ self.assertEqual(record['year'], 1799)
+ self.assertEqual(record['advisors'], set([18230]))
+ self.assertEqual(record['descendants'], set([18603, 18233, 62547,
+ 29642, 55175, 29458,
+ 19953, 18232, 151876]))
def test_get_record_from_tree_bad(self):
"""Verify exception thrown from get_record_from_tree() method for bad
@@ -59,25 +60,27 @@ class TestGrabberMethods(unittest.TestCase):
fields."""
with open(self.data_file('18231.html'), 'r') as fin:
soup = BeautifulSoup(fin, convertEntities='html')
- [name, institution, year, advisors,
- descendents] = get_record_from_tree(soup, 18231)
- self.assertEqual(name, u"Carl Friedrich Gau\xdf")
- self.assertEqual(institution, u"Universit\xe4t Helmstedt")
- self.assertEqual(year, 1799)
- self.assertEqual(advisors, set([18230]))
- self.assertEqual(descendents, set([18603, 18233, 62547, 29642, 55175,
- 29458, 19953, 18232, 151876]))
+ record = get_record_from_tree(soup, 18231)
+ self.assertEqual(len(record), 5)
+ self.assertEqual(record['name'], u"Carl Friedrich Gau\xdf")
+ self.assertEqual(record['institution'], u"Universit\xe4t Helmstedt")
+ self.assertEqual(record['year'], 1799)
+ self.assertEqual(record['advisors'], set([18230]))
+ self.assertEqual(record['descendants'], set([18603, 18233, 62547,
+ 29642, 55175, 29458,
+ 19953, 18232, 151876]))
# Verify calling get_record_from_tree() twice does not have side
# effect.
- [name, institution, year, advisors,
- descendents] = get_record_from_tree(soup, 18231)
- self.assertEqual(name, u"Carl Friedrich Gau\xdf")
- self.assertEqual(institution, u"Universit\xe4t Helmstedt")
- self.assertEqual(year, 1799)
- self.assertEqual(advisors, set([18230]))
- self.assertEqual(descendents, set([18603, 18233, 62547, 29642, 55175,
- 29458, 19953, 18232, 151876]))
+ record = get_record_from_tree(soup, 18231)
+ self.assertEqual(len(record), 5)
+ self.assertEqual(record['name'], u"Carl Friedrich Gau\xdf")
+ self.assertEqual(record['institution'], u"Universit\xe4t Helmstedt")
+ self.assertEqual(record['year'], 1799)
+ self.assertEqual(record['advisors'], set([18230]))
+ self.assertEqual(record['descendants'], set([18603, 18233, 62547,
+ 29642, 55175, 29458,
+ 19953, 18232, 151876]))
def test_has_record_true(self):
"""Test the has_record() method with a tree containing a
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/geneagrapher.git
More information about the debian-science-commits
mailing list