From 435b5371e67fbcd26379524bfb8fd74a20fd68aa Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Fri, 13 Nov 2020 21:23:16 -0500 Subject: [PATCH 001/175] Python3 conversion - setup.py, json config load, and cPickle - change setup.py to stop install if trying to use python 2 - read config as "text" so json.loads does not complain about "bytes" - "cPickle" should now be referenced via standard "pickle" import. See: https://docs.python.org/3.1/whatsnew/3.0.html#library-changes > A common pattern in Python 2.x is to have one version of a module implemented in pure Python, > with an optional accelerated version implemented as a C extension; for example, pickle and cPickle. > This places the burden of importing the accelerated version and falling back on the pure Python > version on each user of these modules. In Python 3.0, the accelerated versions are considered > implementation details of the pure Python versions. Users should always import the standard version, > which attempts to import the accelerated version and falls back to the pure Python version. The > pickle / cPickle pair received this treatment. --- idb/config.py | 2 +- idb/helpers/memoize.py | 22 +++++++++++----------- setup.py | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/idb/config.py b/idb/config.py index 35feabe7..a2f99755 100644 --- a/idb/config.py +++ b/idb/config.py @@ -33,7 +33,7 @@ def update_environment(env): globals()[k] = v def load_config_file(p): - with open(p, "rb") as conf: + with open(p, "r") as conf: logger.debug("Reading config from %r", p) json_config = json.load(conf) config.update(json_config) diff --git a/idb/helpers/memoize.py b/idb/helpers/memoize.py index 8787f95c..ef313480 100644 --- a/idb/helpers/memoize.py +++ b/idb/helpers/memoize.py @@ -2,7 +2,7 @@ from __future__ import print_function from functools import wraps -import cPickle +import pickle from idb.helpers.logging import getLogger from atomicfile import AtomicFile @@ -54,15 +54,15 @@ def memo(*args, **kwargs): return memo -def _memoize_nargs_cPickle(fn): +def _memoize_nargs_pickle(fn): "A memoizer for function w/ arbitrary length arguments." memory = {} - import cPickle + import pickle @wraps(fn) def memo(*args, **kwargs): - key = (cPickle.dumps(args, cPickle.HIGHEST_PROTOCOL), - cPickle.dumps(kwargs, cPickle.HIGHEST_PROTOCOL)) + key = (pickle.dumps(args, pickle.HIGHEST_PROTOCOL), + pickle.dumps(kwargs, pickle.HIGHEST_PROTOCOL)) try: v = memory[key] except KeyError: @@ -72,20 +72,20 @@ def memo(*args, **kwargs): return memo -def memoized(unhashable="cPickle"): +def memoized(unhashable="pickle"): "Decorator to memoize a function." def getfn(fn): if fn.func_code.co_argcount == 0: memo = _memoize_0args(fn) elif unhashable == "call": memo = _memoize_nargs_callthru(fn) - elif unhashable == "cPickle": - memo = _memoize_nargs_cPickle(fn) + elif unhashable == "pickle": + memo = _memoize_nargs_pickle(fn) elif unhashable == "error": memo = _memoize_nargs_error(fn) else: raise ValueError( - "Uknown unhashable memoization strategy, expected {call, cPickle, error}") + "Uknown unhashable memoization strategy, expected {call, pickle, error}") if memo.__doc__: memo.__doc__ = memo.__doc__ + "\n\nThis function is memoized." else: @@ -101,7 +101,7 @@ def filecached(filename, writeback=True): def writecache(value): logger.debug("Writing cache to %r", filename) with AtomicFile(filename, 'wb') as f: - cPickle.dump(value, f) + pickle.dump(value, f) def getfn(fn): @wraps(fn) @@ -109,7 +109,7 @@ def thunk(*args, **kwargs): val = None try: with open(filename, 'rb') as f: - val = cPickle.load(f) + val = pickle.load(f) logger.debug("read cache from %r", filename) except (IOError, EOFError): val = fn(*args, **kwargs) diff --git a/setup.py b/setup.py index 0651377d..be57a633 100644 --- a/setup.py +++ b/setup.py @@ -16,8 +16,8 @@ def read(*paths): read(os.path.join(os.path.dirname(__file__), 'idb/__init__.py')), re.MULTILINE).group(1) # Abort if on Python 3 since we know the codebase does not support it. -if sys.version_info >= (3,0): - sys.exit("idb-backend: Python 3 is not supported. Consider passing '-p python2.7' when creating the venv.") +if sys.version_info < (3,0): + sys.exit("idb-backend: Python 2 is not supported in this branch.") # Pillow-SIMD causes segfault on newer Python 2. # We do not yet support Python 3. From d872138602d31c6da67818ceb1174e2a899bbd73 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 14 Dec 2020 19:47:00 -0500 Subject: [PATCH 002/175] Python3 conversion - setup.py, json config load, cPickle, and print_function - change setup.py to stop install if trying to use python 2 - read config as "text" so json.loads does not complain about "bytes" - "cPickle" should now be referenced via standard "pickle" import. See: https://docs.python.org/3.1/whatsnew/3.0.html#library-changes > A common pattern in Python 2.x is to have one version of a module implemented in pure Python, > with an optional accelerated version implemented as a C extension; for example, pickle and cPickle. > This places the burden of importing the accelerated version and falling back on the pure Python > version on each user of these modules. In Python 3.0, the accelerated versions are considered > implementation details of the pure Python versions. Users should always import the standard version, > which attempts to import the accelerated version and falls back to the pure Python version. The > pickle / cPickle pair received this treatment. - use print function from __future__ - use correct format for date spec --- idb/config.py | 2 +- idb/data_tables/rights_strings.py | 9 ++++---- idb/data_tables/taxon_rank.py | 9 ++++---- idb/helpers/biodiversity_socket_connector.py | 8 +++---- idb/helpers/conversions.py | 6 +++--- idb/helpers/etags.py | 4 ++-- idb/helpers/memoize.py | 22 ++++++++++---------- idigbio_ingestion/lib/eml.py | 3 ++- setup.py | 4 ++-- tests/idb/test_helpers_conversions.py | 2 +- 10 files changed, 36 insertions(+), 33 deletions(-) diff --git a/idb/config.py b/idb/config.py index 35feabe7..a2f99755 100644 --- a/idb/config.py +++ b/idb/config.py @@ -33,7 +33,7 @@ def update_environment(env): globals()[k] = v def load_config_file(p): - with open(p, "rb") as conf: + with open(p, "r") as conf: logger.debug("Reading config from %r", p) json_config = json.load(conf) config.update(json_config) diff --git a/idb/data_tables/rights_strings.py b/idb/data_tables/rights_strings.py index 1c58d440..7112a9fb 100644 --- a/idb/data_tables/rights_strings.py +++ b/idb/data_tables/rights_strings.py @@ -1,3 +1,4 @@ +from __future__ import print_function import re import traceback @@ -230,7 +231,7 @@ def pick_license(s, debug=False): r = strip_special.sub("", m[0]).upper() v = m[-1] if debug: - print r, v, rights_order.index(r), order, picked + print (r, v, rights_order.index(r), order, picked) if rights_order.index(r) > order: if r in ["CC0", "ZERO", "PUBLICDOMAIN"]: picked = rights_strings[r] @@ -241,7 +242,7 @@ def pick_license(s, debug=False): else: pass if debug: - print r, v, rights_order.index(r), order, picked + print (r, v, rights_order.index(r), order, picked) except: traceback.print_exc() return picked @@ -256,6 +257,6 @@ def main(): picked = pick_license(s) if acceptable_licenses_trans[s] != picked: # print s, acceptable_licenses_trans[s], picked - print - print s, acceptable_licenses_trans[s] + print() + print (s, acceptable_licenses_trans[s]) pick_license(s, debug=True) diff --git a/idb/data_tables/taxon_rank.py b/idb/data_tables/taxon_rank.py index bc483bb9..0a965d2e 100644 --- a/idb/data_tables/taxon_rank.py +++ b/idb/data_tables/taxon_rank.py @@ -1,4 +1,5 @@ # -*- coding: UTF-8 -*- +from __future__ import print_function # http://rs.gbif.org/vocabulary/gbif/rank.xml acceptable = { @@ -377,17 +378,17 @@ def main(): o = r.json() for k in o["taxonrank"]: if k not in mapping: - print k, o["taxonrank"][k] + print (k, o["taxonrank"][k]) elif mapping[k] is None: pass elif mapping[k] not in acceptable: - print k, mapping[k] + print (k, mapping[k]) new_accept_set.add(mapping[k]) if len(new_accept_set) > 0: - print json.dumps(list(new_accept_set), indent=4) + print (json.dumps(list(new_accept_set), indent=4)) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/idb/helpers/biodiversity_socket_connector.py b/idb/helpers/biodiversity_socket_connector.py index 2c6a4482..6f6b01ee 100644 --- a/idb/helpers/biodiversity_socket_connector.py +++ b/idb/helpers/biodiversity_socket_connector.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function import socket import json @@ -101,13 +101,13 @@ def get_genus_species_list(self, namestr_list): def main(): b = Biodiversity() - print b.get_genus_species("Puma concolor") + print (b.get_genus_species("Puma concolor")) - print b.get_genus_species_list([ + print (b.get_genus_species_list([ "Puma concolor", "Quercus alba", "Acer floridanum" - ]) + ])) if __name__ == '__main__': main() diff --git a/idb/helpers/conversions.py b/idb/helpers/conversions.py index 626f8b65..61354fcb 100644 --- a/idb/helpers/conversions.py +++ b/idb/helpers/conversions.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function import dateutil.parser import re import traceback @@ -991,7 +991,7 @@ def dt_serial(obj): continue break - print "records ready" + print ("records ready") interations = 1 @@ -1009,7 +1009,7 @@ def dt_serial(obj): total_time += (t2 - t1).total_seconds() count += 1 - print count, total_time, total_time / count + print (count, total_time, total_time / count) # t = sys.argv[1] # u = sys.argv[2] diff --git a/idb/helpers/etags.py b/idb/helpers/etags.py index 4ab7228b..9e49ce27 100644 --- a/idb/helpers/etags.py +++ b/idb/helpers/etags.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function import hashlib import json @@ -69,7 +69,7 @@ def objectHasher(hash_type, data, sort_arrays=False, sort_keys=True): elif data is None: pass else: - print type(data) + print (type(data)) # print s h.update(s.encode("utf8")) diff --git a/idb/helpers/memoize.py b/idb/helpers/memoize.py index 8787f95c..ef313480 100644 --- a/idb/helpers/memoize.py +++ b/idb/helpers/memoize.py @@ -2,7 +2,7 @@ from __future__ import print_function from functools import wraps -import cPickle +import pickle from idb.helpers.logging import getLogger from atomicfile import AtomicFile @@ -54,15 +54,15 @@ def memo(*args, **kwargs): return memo -def _memoize_nargs_cPickle(fn): +def _memoize_nargs_pickle(fn): "A memoizer for function w/ arbitrary length arguments." memory = {} - import cPickle + import pickle @wraps(fn) def memo(*args, **kwargs): - key = (cPickle.dumps(args, cPickle.HIGHEST_PROTOCOL), - cPickle.dumps(kwargs, cPickle.HIGHEST_PROTOCOL)) + key = (pickle.dumps(args, pickle.HIGHEST_PROTOCOL), + pickle.dumps(kwargs, pickle.HIGHEST_PROTOCOL)) try: v = memory[key] except KeyError: @@ -72,20 +72,20 @@ def memo(*args, **kwargs): return memo -def memoized(unhashable="cPickle"): +def memoized(unhashable="pickle"): "Decorator to memoize a function." def getfn(fn): if fn.func_code.co_argcount == 0: memo = _memoize_0args(fn) elif unhashable == "call": memo = _memoize_nargs_callthru(fn) - elif unhashable == "cPickle": - memo = _memoize_nargs_cPickle(fn) + elif unhashable == "pickle": + memo = _memoize_nargs_pickle(fn) elif unhashable == "error": memo = _memoize_nargs_error(fn) else: raise ValueError( - "Uknown unhashable memoization strategy, expected {call, cPickle, error}") + "Uknown unhashable memoization strategy, expected {call, pickle, error}") if memo.__doc__: memo.__doc__ = memo.__doc__ + "\n\nThis function is memoized." else: @@ -101,7 +101,7 @@ def filecached(filename, writeback=True): def writecache(value): logger.debug("Writing cache to %r", filename) with AtomicFile(filename, 'wb') as f: - cPickle.dump(value, f) + pickle.dump(value, f) def getfn(fn): @wraps(fn) @@ -109,7 +109,7 @@ def thunk(*args, **kwargs): val = None try: with open(filename, 'rb') as f: - val = cPickle.load(f) + val = pickle.load(f) logger.debug("read cache from %r", filename) except (IOError, EOFError): val = fn(*args, **kwargs) diff --git a/idigbio_ingestion/lib/eml.py b/idigbio_ingestion/lib/eml.py index 59cc2b00..dcb30b8d 100644 --- a/idigbio_ingestion/lib/eml.py +++ b/idigbio_ingestion/lib/eml.py @@ -1,3 +1,4 @@ +from __future__ import print_function from pyquery import PyQuery as pq from idb.data_tables.rights_strings import acceptable_licenses_trans @@ -155,7 +156,7 @@ def main(): import sys import json with open(sys.argv[1],"rb") as inf: - print json.dumps(parseEml("testid",inf.read())) + print (json.dumps(parseEml("testid",inf.read()))) if __name__ == '__main__': diff --git a/setup.py b/setup.py index 0651377d..be57a633 100644 --- a/setup.py +++ b/setup.py @@ -16,8 +16,8 @@ def read(*paths): read(os.path.join(os.path.dirname(__file__), 'idb/__init__.py')), re.MULTILINE).group(1) # Abort if on Python 3 since we know the codebase does not support it. -if sys.version_info >= (3,0): - sys.exit("idb-backend: Python 3 is not supported. Consider passing '-p python2.7' when creating the venv.") +if sys.version_info < (3,0): + sys.exit("idb-backend: Python 2 is not supported in this branch.") # Pillow-SIMD causes segfault on newer Python 2. # We do not yet support Python 3. diff --git a/tests/idb/test_helpers_conversions.py b/tests/idb/test_helpers_conversions.py index 3098b598..f26fdc0d 100644 --- a/tests/idb/test_helpers_conversions.py +++ b/tests/idb/test_helpers_conversions.py @@ -208,7 +208,7 @@ def test_date_grabber_year_month_day_fallback(self): } self.assertEqual({ "datemodified": datetime.datetime(2014, 1, 10, tzinfo=pytz.utc), - "datecollected": datetime.date(2014,01,10), + "datecollected": datetime.date(2014, 1, 10), "startdayofyear": 10, }, conversions.dateGrabber("records", r)) From 92853eed1501e6849bfecdfd3458451da926794e Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 14 Dec 2020 20:03:50 -0500 Subject: [PATCH 003/175] replace func_code with __code__ --- idb/helpers/memoize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idb/helpers/memoize.py b/idb/helpers/memoize.py index ef313480..2f110c48 100644 --- a/idb/helpers/memoize.py +++ b/idb/helpers/memoize.py @@ -75,7 +75,7 @@ def memo(*args, **kwargs): def memoized(unhashable="pickle"): "Decorator to memoize a function." def getfn(fn): - if fn.func_code.co_argcount == 0: + if fn.__code__.co_argcount == 0: memo = _memoize_0args(fn) elif unhashable == "call": memo = _memoize_nargs_callthru(fn) From cb00c6f077ab50bb57d71fefb0038fbcc0f20f6a Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 14 Dec 2020 20:04:37 -0500 Subject: [PATCH 004/175] fix imports for email module --- idigbio_workers/lib/mailer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/idigbio_workers/lib/mailer.py b/idigbio_workers/lib/mailer.py index 2c8b215d..8380d5ad 100644 --- a/idigbio_workers/lib/mailer.py +++ b/idigbio_workers/lib/mailer.py @@ -1,11 +1,11 @@ from __future__ import division, absolute_import, print_function import os.path import smtplib -from email.MIMEMultipart import MIMEMultipart -from email.MIMEBase import MIMEBase -from email.MIMEText import MIMEText -from email.Utils import COMMASPACE, formatdate -from email import Encoders +from email.mime.multipart import MIMEMultipart +from email.mime.base import MIMEBase +from email.mime.text import MIMEText +from email.utils import COMMASPACE, formatdate +from email import encoders from idb.helpers.logging import idblogger From a57c87691cecf5730622f86bbe6c76e28f8289f2 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 14 Dec 2020 19:47:00 -0500 Subject: [PATCH 005/175] Python3 conversion - setup.py, json config load, cPickle, and print_function - change setup.py to stop install if trying to use python 2 - read config as "text" so json.loads does not complain about "bytes" - "cPickle" should now be referenced via standard "pickle" import. See: https://docs.python.org/3.1/whatsnew/3.0.html#library-changes > A common pattern in Python 2.x is to have one version of a module implemented in pure Python, > with an optional accelerated version implemented as a C extension; for example, pickle and cPickle. > This places the burden of importing the accelerated version and falling back on the pure Python > version on each user of these modules. In Python 3.0, the accelerated versions are considered > implementation details of the pure Python versions. Users should always import the standard version, > which attempts to import the accelerated version and falls back to the pure Python version. The > pickle / cPickle pair received this treatment. - use print function from __future__ - use correct format for date spec --- idb/config.py | 2 +- idb/data_tables/rights_strings.py | 9 ++++---- idb/data_tables/taxon_rank.py | 9 ++++---- idb/helpers/biodiversity_socket_connector.py | 8 +++---- idb/helpers/conversions.py | 6 +++--- idb/helpers/etags.py | 4 ++-- idb/helpers/memoize.py | 22 ++++++++++---------- idigbio_ingestion/lib/eml.py | 3 ++- setup.py | 4 ++-- tests/idb/test_helpers_conversions.py | 2 +- 10 files changed, 36 insertions(+), 33 deletions(-) diff --git a/idb/config.py b/idb/config.py index 35feabe7..a2f99755 100644 --- a/idb/config.py +++ b/idb/config.py @@ -33,7 +33,7 @@ def update_environment(env): globals()[k] = v def load_config_file(p): - with open(p, "rb") as conf: + with open(p, "r") as conf: logger.debug("Reading config from %r", p) json_config = json.load(conf) config.update(json_config) diff --git a/idb/data_tables/rights_strings.py b/idb/data_tables/rights_strings.py index 1c58d440..7112a9fb 100644 --- a/idb/data_tables/rights_strings.py +++ b/idb/data_tables/rights_strings.py @@ -1,3 +1,4 @@ +from __future__ import print_function import re import traceback @@ -230,7 +231,7 @@ def pick_license(s, debug=False): r = strip_special.sub("", m[0]).upper() v = m[-1] if debug: - print r, v, rights_order.index(r), order, picked + print (r, v, rights_order.index(r), order, picked) if rights_order.index(r) > order: if r in ["CC0", "ZERO", "PUBLICDOMAIN"]: picked = rights_strings[r] @@ -241,7 +242,7 @@ def pick_license(s, debug=False): else: pass if debug: - print r, v, rights_order.index(r), order, picked + print (r, v, rights_order.index(r), order, picked) except: traceback.print_exc() return picked @@ -256,6 +257,6 @@ def main(): picked = pick_license(s) if acceptable_licenses_trans[s] != picked: # print s, acceptable_licenses_trans[s], picked - print - print s, acceptable_licenses_trans[s] + print() + print (s, acceptable_licenses_trans[s]) pick_license(s, debug=True) diff --git a/idb/data_tables/taxon_rank.py b/idb/data_tables/taxon_rank.py index bc483bb9..0a965d2e 100644 --- a/idb/data_tables/taxon_rank.py +++ b/idb/data_tables/taxon_rank.py @@ -1,4 +1,5 @@ # -*- coding: UTF-8 -*- +from __future__ import print_function # http://rs.gbif.org/vocabulary/gbif/rank.xml acceptable = { @@ -377,17 +378,17 @@ def main(): o = r.json() for k in o["taxonrank"]: if k not in mapping: - print k, o["taxonrank"][k] + print (k, o["taxonrank"][k]) elif mapping[k] is None: pass elif mapping[k] not in acceptable: - print k, mapping[k] + print (k, mapping[k]) new_accept_set.add(mapping[k]) if len(new_accept_set) > 0: - print json.dumps(list(new_accept_set), indent=4) + print (json.dumps(list(new_accept_set), indent=4)) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/idb/helpers/biodiversity_socket_connector.py b/idb/helpers/biodiversity_socket_connector.py index 2c6a4482..6f6b01ee 100644 --- a/idb/helpers/biodiversity_socket_connector.py +++ b/idb/helpers/biodiversity_socket_connector.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function import socket import json @@ -101,13 +101,13 @@ def get_genus_species_list(self, namestr_list): def main(): b = Biodiversity() - print b.get_genus_species("Puma concolor") + print (b.get_genus_species("Puma concolor")) - print b.get_genus_species_list([ + print (b.get_genus_species_list([ "Puma concolor", "Quercus alba", "Acer floridanum" - ]) + ])) if __name__ == '__main__': main() diff --git a/idb/helpers/conversions.py b/idb/helpers/conversions.py index 626f8b65..61354fcb 100644 --- a/idb/helpers/conversions.py +++ b/idb/helpers/conversions.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function import dateutil.parser import re import traceback @@ -991,7 +991,7 @@ def dt_serial(obj): continue break - print "records ready" + print ("records ready") interations = 1 @@ -1009,7 +1009,7 @@ def dt_serial(obj): total_time += (t2 - t1).total_seconds() count += 1 - print count, total_time, total_time / count + print (count, total_time, total_time / count) # t = sys.argv[1] # u = sys.argv[2] diff --git a/idb/helpers/etags.py b/idb/helpers/etags.py index 4ab7228b..9e49ce27 100644 --- a/idb/helpers/etags.py +++ b/idb/helpers/etags.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function import hashlib import json @@ -69,7 +69,7 @@ def objectHasher(hash_type, data, sort_arrays=False, sort_keys=True): elif data is None: pass else: - print type(data) + print (type(data)) # print s h.update(s.encode("utf8")) diff --git a/idb/helpers/memoize.py b/idb/helpers/memoize.py index 8787f95c..ef313480 100644 --- a/idb/helpers/memoize.py +++ b/idb/helpers/memoize.py @@ -2,7 +2,7 @@ from __future__ import print_function from functools import wraps -import cPickle +import pickle from idb.helpers.logging import getLogger from atomicfile import AtomicFile @@ -54,15 +54,15 @@ def memo(*args, **kwargs): return memo -def _memoize_nargs_cPickle(fn): +def _memoize_nargs_pickle(fn): "A memoizer for function w/ arbitrary length arguments." memory = {} - import cPickle + import pickle @wraps(fn) def memo(*args, **kwargs): - key = (cPickle.dumps(args, cPickle.HIGHEST_PROTOCOL), - cPickle.dumps(kwargs, cPickle.HIGHEST_PROTOCOL)) + key = (pickle.dumps(args, pickle.HIGHEST_PROTOCOL), + pickle.dumps(kwargs, pickle.HIGHEST_PROTOCOL)) try: v = memory[key] except KeyError: @@ -72,20 +72,20 @@ def memo(*args, **kwargs): return memo -def memoized(unhashable="cPickle"): +def memoized(unhashable="pickle"): "Decorator to memoize a function." def getfn(fn): if fn.func_code.co_argcount == 0: memo = _memoize_0args(fn) elif unhashable == "call": memo = _memoize_nargs_callthru(fn) - elif unhashable == "cPickle": - memo = _memoize_nargs_cPickle(fn) + elif unhashable == "pickle": + memo = _memoize_nargs_pickle(fn) elif unhashable == "error": memo = _memoize_nargs_error(fn) else: raise ValueError( - "Uknown unhashable memoization strategy, expected {call, cPickle, error}") + "Uknown unhashable memoization strategy, expected {call, pickle, error}") if memo.__doc__: memo.__doc__ = memo.__doc__ + "\n\nThis function is memoized." else: @@ -101,7 +101,7 @@ def filecached(filename, writeback=True): def writecache(value): logger.debug("Writing cache to %r", filename) with AtomicFile(filename, 'wb') as f: - cPickle.dump(value, f) + pickle.dump(value, f) def getfn(fn): @wraps(fn) @@ -109,7 +109,7 @@ def thunk(*args, **kwargs): val = None try: with open(filename, 'rb') as f: - val = cPickle.load(f) + val = pickle.load(f) logger.debug("read cache from %r", filename) except (IOError, EOFError): val = fn(*args, **kwargs) diff --git a/idigbio_ingestion/lib/eml.py b/idigbio_ingestion/lib/eml.py index 59cc2b00..dcb30b8d 100644 --- a/idigbio_ingestion/lib/eml.py +++ b/idigbio_ingestion/lib/eml.py @@ -1,3 +1,4 @@ +from __future__ import print_function from pyquery import PyQuery as pq from idb.data_tables.rights_strings import acceptable_licenses_trans @@ -155,7 +156,7 @@ def main(): import sys import json with open(sys.argv[1],"rb") as inf: - print json.dumps(parseEml("testid",inf.read())) + print (json.dumps(parseEml("testid",inf.read()))) if __name__ == '__main__': diff --git a/setup.py b/setup.py index 0651377d..be57a633 100644 --- a/setup.py +++ b/setup.py @@ -16,8 +16,8 @@ def read(*paths): read(os.path.join(os.path.dirname(__file__), 'idb/__init__.py')), re.MULTILINE).group(1) # Abort if on Python 3 since we know the codebase does not support it. -if sys.version_info >= (3,0): - sys.exit("idb-backend: Python 3 is not supported. Consider passing '-p python2.7' when creating the venv.") +if sys.version_info < (3,0): + sys.exit("idb-backend: Python 2 is not supported in this branch.") # Pillow-SIMD causes segfault on newer Python 2. # We do not yet support Python 3. diff --git a/tests/idb/test_helpers_conversions.py b/tests/idb/test_helpers_conversions.py index 3098b598..f26fdc0d 100644 --- a/tests/idb/test_helpers_conversions.py +++ b/tests/idb/test_helpers_conversions.py @@ -208,7 +208,7 @@ def test_date_grabber_year_month_day_fallback(self): } self.assertEqual({ "datemodified": datetime.datetime(2014, 1, 10, tzinfo=pytz.utc), - "datecollected": datetime.date(2014,01,10), + "datecollected": datetime.date(2014, 1, 10), "startdayofyear": 10, }, conversions.dateGrabber("records", r)) From dbfd755f084a0541c18949e2c6eca12bf719bac5 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 14 Dec 2020 20:03:50 -0500 Subject: [PATCH 006/175] replace func_code with __code__ --- idb/helpers/memoize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idb/helpers/memoize.py b/idb/helpers/memoize.py index ef313480..2f110c48 100644 --- a/idb/helpers/memoize.py +++ b/idb/helpers/memoize.py @@ -75,7 +75,7 @@ def memo(*args, **kwargs): def memoized(unhashable="pickle"): "Decorator to memoize a function." def getfn(fn): - if fn.func_code.co_argcount == 0: + if fn.__code__.co_argcount == 0: memo = _memoize_0args(fn) elif unhashable == "call": memo = _memoize_nargs_callthru(fn) From 90525b7231dd3dfb39d144ec40f79137026d6257 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 14 Dec 2020 20:04:37 -0500 Subject: [PATCH 007/175] fix imports for email module --- idigbio_workers/lib/mailer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/idigbio_workers/lib/mailer.py b/idigbio_workers/lib/mailer.py index 2c8b215d..8380d5ad 100644 --- a/idigbio_workers/lib/mailer.py +++ b/idigbio_workers/lib/mailer.py @@ -1,11 +1,11 @@ from __future__ import division, absolute_import, print_function import os.path import smtplib -from email.MIMEMultipart import MIMEMultipart -from email.MIMEBase import MIMEBase -from email.MIMEText import MIMEText -from email.Utils import COMMASPACE, formatdate -from email import Encoders +from email.mime.multipart import MIMEMultipart +from email.mime.base import MIMEBase +from email.mime.text import MIMEText +from email.utils import COMMASPACE, formatdate +from email import encoders from idb.helpers.logging import idblogger From 194cdf8bbfb7bf92d0b9f2462ca40272bd4799d6 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 12 Jan 2021 15:46:37 -0500 Subject: [PATCH 008/175] use raw strings so regex escape sequence is valid To resolve deprecation warnings about invalid escape sequences: DeprecationWarning: invalid escape sequence which occur when trying to use single backslashes in string literals. --- idb/helpers/conversions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/idb/helpers/conversions.py b/idb/helpers/conversions.py index 61354fcb..47215b51 100644 --- a/idb/helpers/conversions.py +++ b/idb/helpers/conversions.py @@ -291,7 +291,7 @@ def verbatimGrabber(t, d): r[f[0]] = getfield(f[1], d, t=f[2]) return r -gfn = re.compile("([+-]?[0-9]+(?:[,][0-9]{3})*(?:[\.][0-9]*)?)") +gfn = re.compile(r"([+-]?[0-9]+(?:[,][0-9]{3})*(?:[\.][0-9]*)?)") def grabFirstNumber(f): @@ -307,7 +307,7 @@ def grabFirstNumber(f): pass return n -mangler = re.compile("[\W]+") +mangler = re.compile(r"[\W]+") def mangleString(s): @@ -315,7 +315,7 @@ def mangleString(s): uuid_re = re.compile( - "([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})") + r"([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})") def grabFirstUUID(f): @@ -812,7 +812,7 @@ def collect_common_names(t, d): return {} -genbank_ids = re.compile("[a-zA-Z]{1,2}\-?_?\d{5,6}") +genbank_ids = re.compile(r"[a-zA-Z]{1,2}\-?_?\d{5,6}") def collect_genbank_sequences(t, d): if t == "records": From f172ec8c1d24c1e6221ec68b4386ccde522546cb Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 12 Jan 2021 16:21:17 -0500 Subject: [PATCH 009/175] update gevent to 1.3.0 To resolve the following deprecation warning: DeprecationWarning: inspect.getargspec() is deprecated since Python 3.0, use inspect.signature() or inspect.getfullargspec() 1.3.0 seems to be the newest version of gevent that both eliminates the deprecation warning and also does not cause other dependency issues (such as requiring a newer greenlet). --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index be57a633..12c1a3f1 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ def read(*paths): 'coverage', 'numpy', 'scipy', - 'gevent>=1.1.0, <1.2.0', + 'gevent=1.3.0', 'gipc>=0.6.0, <0.7.0', 'unicodecsv>=0.14.1, < 0.15.0', 'shapely', From 7ce689b15e51ce8d351cdb740b3bd8c9c74a6452 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 12 Jan 2021 20:00:36 -0500 Subject: [PATCH 010/175] use raw string so regex escape sequence is valid --- idb/data_tables/rights_strings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idb/data_tables/rights_strings.py b/idb/data_tables/rights_strings.py index 7112a9fb..25f1d2fd 100644 --- a/idb/data_tables/rights_strings.py +++ b/idb/data_tables/rights_strings.py @@ -222,7 +222,7 @@ def pick_license(s, debug=False): return manual_assignment[s] rights_regex = re.compile( - "((?:by(?:.?nc)?(?:.?sa)?(?:.?nd)?)|cc0|zero|(?:public.?domain)).?(\d\.\d)?", re.I) + r"((?:by(?:.?nc)?(?:.?sa)?(?:.?nd)?)|cc0|zero|(?:public.?domain)).?(\d\.\d)?", re.I) strip_special = re.compile('[^0-9a-zA-Z]+') picked = None order = -1 From b67f42d48cb5148525f2e4128b671082c615da28 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 12 Jan 2021 20:09:36 -0500 Subject: [PATCH 011/175] fix syntax error --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 12c1a3f1..047bd34a 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ def read(*paths): 'coverage', 'numpy', 'scipy', - 'gevent=1.3.0', + 'gevent==1.3.0', 'gipc>=0.6.0, <0.7.0', 'unicodecsv>=0.14.1, < 0.15.0', 'shapely', From c26a66cd8bce7a320eb779e6e97242c522119215 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 12 Jan 2021 21:12:42 -0500 Subject: [PATCH 012/175] StandardError exception name was removed in python3 --- idigbio_ingestion/mediaing/fetcher.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/idigbio_ingestion/mediaing/fetcher.py b/idigbio_ingestion/mediaing/fetcher.py index 697037e4..3e59f7c6 100644 --- a/idigbio_ingestion/mediaing/fetcher.py +++ b/idigbio_ingestion/mediaing/fetcher.py @@ -117,7 +117,8 @@ def process_list(fetchitems, forprefix=''): items = update_db_status(items) items = count_result_types(items, forprefix=forprefix) return ilen(items) # consume the generator - except StandardError: + + except: logger.exception("Unhandled error forprefix:%s", forprefix) raise From 500f85d8db62aa14f3549cfa32d1dd3aa1c3ce81 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 12 Jan 2021 21:14:56 -0500 Subject: [PATCH 013/175] python3 compatible imports, modules, methods --- idigbio_ingestion/mediaing/derivatives.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/idigbio_ingestion/mediaing/derivatives.py b/idigbio_ingestion/mediaing/derivatives.py index ebef3b55..34f45b3e 100644 --- a/idigbio_ingestion/mediaing/derivatives.py +++ b/idigbio_ingestion/mediaing/derivatives.py @@ -1,12 +1,10 @@ from __future__ import division, absolute_import from __future__ import print_function -from future_builtins import map, filter -import cStringIO +from io import BytesIO from datetime import datetime from collections import Counter, namedtuple -import itertools from gevent.pool import Pool from PIL import Image @@ -87,7 +85,7 @@ def one(o): ci = get_keys(o) gr = generate_all(ci) return upload_all(gr) - results = pool.imap_unordered(one, itertools.ifilter(None, objects)) + results = pool.imap_unordered(one, filter(None, objects)) results = count_results(results, update_freq=100) etags = ((gr.etag,) for gr in results if gr) count = apidbpool.executemany( @@ -151,7 +149,7 @@ def output(): def get_keys(obj): etag, bucket = obj.etag, obj.bucket - etag = unicode(etag) + etag = str(etag) s = get_store() bucketbase = u"idigbio-{0}-{1}".format(bucket, config.ENV) mediakey = s.get_key(etag, bucketbase) @@ -235,7 +233,7 @@ def upload_item(item): def img_to_buffer(img, **kwargs): kwargs.setdefault('format', 'JPEG') kwargs.setdefault('quality', 95) - dervbuff = cStringIO.StringIO() + dervbuff = BytesIO() img.save(dervbuff, **kwargs) dervbuff.seek(0) return dervbuff From c33094065840c9fd0eac0c7bb61211c43f10e33a Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 12 Jan 2021 21:17:04 -0500 Subject: [PATCH 014/175] python3 no longer has a basestring type --- idb/data_api/v2_download.py | 2 +- idb/helpers/cors.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/idb/data_api/v2_download.py b/idb/data_api/v2_download.py index f5f59c52..4dcac90a 100644 --- a/idb/data_api/v2_download.py +++ b/idb/data_api/v2_download.py @@ -55,7 +55,7 @@ def download(): if isinstance(o[k], list): o[k] = o[k][0] - if isinstance(o[k], basestring): + if isinstance(o[k], str): try: params[k] = json.loads(o[k]) except ValueError: diff --git a/idb/helpers/cors.py b/idb/helpers/cors.py index 3d79fb45..5b6db26e 100644 --- a/idb/helpers/cors.py +++ b/idb/helpers/cors.py @@ -9,9 +9,9 @@ def crossdomain(origin=None, methods=None, headers=None, automatic_options=True): if methods is not None: methods = ', '.join(sorted(x.upper() for x in methods)) - if headers is not None and not isinstance(headers, basestring): + if headers is not None and not isinstance(headers, str): headers = ', '.join(x.upper() for x in headers) - if not isinstance(origin, basestring): + if not isinstance(origin, str): origin = ', '.join(origin) if isinstance(max_age, timedelta): max_age = max_age.total_seconds() From 78e655d5e1872cffe07609958ad5ccf9655da75b Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 12 Jan 2021 21:21:44 -0500 Subject: [PATCH 015/175] basestring, unicode no longer exist in python3 --- idb/helpers/storage.py | 2 +- tests/idigbio_ingestion/mediaing/test_derivatives.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/idb/helpers/storage.py b/idb/helpers/storage.py index ea1f25e7..18cdcf94 100644 --- a/idb/helpers/storage.py +++ b/idb/helpers/storage.py @@ -102,7 +102,7 @@ def upload(self, key, fobj, content_type=None, public=None, md5=None, multipart= pass elif hasattr(fobj, 'open'): fobj = fobj.open('rb') - elif isinstance(fobj, basestring): + elif isinstance(fobj, str): fobj = open(fobj, 'rb') elif not all(hasattr(fobj, a) for a in ('seek', 'tell', 'read')): raise ValueError("Unknown fobj type:", fobj) diff --git a/tests/idigbio_ingestion/mediaing/test_derivatives.py b/tests/idigbio_ingestion/mediaing/test_derivatives.py index d20145d3..6334ef01 100644 --- a/tests/idigbio_ingestion/mediaing/test_derivatives.py +++ b/tests/idigbio_ingestion/mediaing/test_derivatives.py @@ -39,7 +39,7 @@ def get_contents_to_file(self, buff): @pytest.fixture() def rand_etag(): - return unicode(uuid.uuid4()) + return str(uuid.uuid4()) @pytest.fixture() From 9a9f22f7920a403e074320e711639c6631ff14b4 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 12 Jan 2021 21:25:06 -0500 Subject: [PATCH 016/175] replace cStringIO --- idb/postgres_backend/db.py | 2 +- idigbio_ingestion/mediaing/fetcher.py | 2 +- idigbio_workers/generate_static_datasets_index.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/idb/postgres_backend/db.py b/idb/postgres_backend/db.py index 72e2959c..5de90596 100644 --- a/idb/postgres_backend/db.py +++ b/idb/postgres_backend/db.py @@ -7,7 +7,7 @@ import itertools from datetime import datetime -from cStringIO import StringIO +from io import StringIO from psycopg2.extras import DictCursor, NamedTupleCursor from psycopg2.extensions import cursor diff --git a/idigbio_ingestion/mediaing/fetcher.py b/idigbio_ingestion/mediaing/fetcher.py index 3e59f7c6..e12015f5 100644 --- a/idigbio_ingestion/mediaing/fetcher.py +++ b/idigbio_ingestion/mediaing/fetcher.py @@ -6,7 +6,7 @@ import signal from collections import Counter from datetime import datetime -from cStringIO import StringIO +from io import StringIO import requests import gevent.pool diff --git a/idigbio_workers/generate_static_datasets_index.py b/idigbio_workers/generate_static_datasets_index.py index 6e2ac9f2..31163956 100644 --- a/idigbio_workers/generate_static_datasets_index.py +++ b/idigbio_workers/generate_static_datasets_index.py @@ -8,7 +8,7 @@ import requests import json import re -from cStringIO import StringIO +from io import StringIO from idb.helpers.storage import IDigBioStorage From 6921a69e3509e21e14ddfe38adc90f7ade7f5d9b Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 12 Jan 2021 21:26:09 -0500 Subject: [PATCH 017/175] add a FIXME and replace itertools with standard lib --- idigbio_workers/lib/download.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/idigbio_workers/lib/download.py b/idigbio_workers/lib/download.py index ccee91e6..1c8e66db 100644 --- a/idigbio_workers/lib/download.py +++ b/idigbio_workers/lib/download.py @@ -5,7 +5,6 @@ import zipfile import os import datetime -import itertools from collections import Counter, namedtuple @@ -276,6 +275,10 @@ def make_file(t, query, raw=False, tabs=False, fields=None, es = get_connection() mapping = es.indices.get_mapping(index=indexName, doc_type=t) + + #FIXME + # Can't get the zeroeth element out of a dictionary anymore. + # https://stackoverflow.com/questions/17431638/get-typeerror-dict-values-object-does-not-support-indexing-when-using-python mapping_root = mapping.values()[0]["mappings"][t]["properties"] if raw: mapping_root = mapping_root["data"]["properties"] @@ -459,7 +462,7 @@ def generate_files(core_type="records", core_source="indexterms", record_query=N zipfilename = filename + ".zip" with zipfile.ZipFile(zipfilename, 'w', zipfile.ZIP_DEFLATED, True) as expzip: meta_files = [] - for fa in itertools.ifilter(None, files): + for fa in filter(None, files): expzip.write(fa.filename, fa.archivename) os.unlink(fa.filename) if fa.meta_block is not None: From cebfd42f960f0b7f7298c33c6a5e51baff468208 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 13 Jan 2021 17:37:35 -0500 Subject: [PATCH 018/175] convert boto md5 bytes to string for comparison key.md5 returns a bytes object, not a string. This leads to a mismatch when bytes are compared to a str. boto.exception.S3DataError: BotoClientError: MD5 of downloaded did not match given MD5: b'bac1d6137adc8974f200173b6a4f05b8' vs. bac1d6137adc8974f200173b6a4f05b8 This seems to be caused by an interaction between 'boto' and 'six'. We are already using the latest version of these libraries. --- idb/helpers/storage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/idb/helpers/storage.py b/idb/helpers/storage.py index 18cdcf94..32b4a351 100644 --- a/idb/helpers/storage.py +++ b/idb/helpers/storage.py @@ -3,7 +3,7 @@ """ from __future__ import division, absolute_import, print_function -import cStringIO +from io import BytesIO import math import os import time @@ -194,8 +194,8 @@ def get_contents_to_filename(key, filename, md5=None): @staticmethod def get_contents_to_mem(key, md5=None): - "Wraps ``key.get_contents_to_file fetching into a StringIO buffer``" - buff = cStringIO.StringIO() + "Wraps ``key.get_contents_to_file fetching into a BytesIO buffer``" + buff = BytesIO() IDigBioStorage.get_contents_to_file(key, buff, md5=md5) buff.seek(0) return buff @@ -203,7 +203,7 @@ def get_contents_to_mem(key, md5=None): @staticmethod def get_contents_to_file(key, fp, md5=None): key.get_contents_to_file(fp) - if md5 and key.md5 != md5: + if md5 is not None and key.md5.decode('utf-8') != md5: raise S3DataError( 'MD5 of downloaded did not match given MD5: ' '%s vs. %s' % (key.md5, md5)) From d4a117ca470fb2755abe8c625a4a5a68d739ceb6 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 13 Jan 2021 18:04:30 -0500 Subject: [PATCH 019/175] replace cStringIO --- tests/idigbio_ingestion/mediaing/test_derivatives.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/idigbio_ingestion/mediaing/test_derivatives.py b/tests/idigbio_ingestion/mediaing/test_derivatives.py index 6334ef01..d0ced69f 100644 --- a/tests/idigbio_ingestion/mediaing/test_derivatives.py +++ b/tests/idigbio_ingestion/mediaing/test_derivatives.py @@ -4,7 +4,7 @@ import pytest import uuid from collections import namedtuple -from cStringIO import StringIO +from io import BytesIO from idigbio_ingestion.mediaing import derivatives @@ -57,6 +57,7 @@ def existing_CheckItem(rand_etag): @pytest.fixture() def nonexisting_CheckItem(rand_etag): + # this "etag" is in the form of a uuid etag = rand_etag bucket = 'images' bucketbase = "idigbio-{0}-{1}".format(bucket, 'test') @@ -165,7 +166,7 @@ def test_generate_all_nonempty(nonexisting_CheckItem, monkeypatch, img): def test_zero_waveform(): - buff = StringIO(b'\xff\xfbT\xc4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00Info\x00\x00\x00\x0f\x00\x00\x00\x02\x00\x00\x02@\x00\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x009LAME3.98r\x01\xa5\x00\x00\x00\x00-\xfe\x00\x00\x14@$\x06\xb8\xc2\x00\x00@\x00\x00\x02@\xfa%]^\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xfbT\xc4\x00\x03\xc0\x00\x01\xa4\x00\x00\x00 \x00\x004\x80\x00\x00\x04LAME3.98.4UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU\xff\xfbT\xc4U\x83\xc0\x00\x01\xa4\x00\x00\x00 \x00\x004\x80\x00\x00\x04UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU') + buff = BytesIO(b'\xff\xfbT\xc4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00Info\x00\x00\x00\x0f\x00\x00\x00\x02\x00\x00\x02@\x00\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x009LAME3.98r\x01\xa5\x00\x00\x00\x00-\xfe\x00\x00\x14@$\x06\xb8\xc2\x00\x00@\x00\x00\x02@\xfa%]^\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xfbT\xc4\x00\x03\xc0\x00\x01\xa4\x00\x00\x00 \x00\x004\x80\x00\x00\x04LAME3.98.4UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU\xff\xfbT\xc4U\x83\xc0\x00\x01\xa4\x00\x00\x00 \x00\x004\x80\x00\x00\x04UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU') img = derivatives.wave_to_img(buff) assert img assert len(derivatives.img_to_buffer(img).getvalue()) From 32ab8353d85709c9e43c06249b3fbdd3a622d301 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 13 Jan 2021 18:58:03 -0500 Subject: [PATCH 020/175] Exception no longer has a "message" in python3 --- idigbio_ingestion/mediaing/derivatives.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idigbio_ingestion/mediaing/derivatives.py b/idigbio_ingestion/mediaing/derivatives.py index 34f45b3e..3329891a 100644 --- a/idigbio_ingestion/mediaing/derivatives.py +++ b/idigbio_ingestion/mediaing/derivatives.py @@ -172,7 +172,7 @@ def generate_all(item): return None # catch PIL.Image.DecompressionBombError and other exceptions here except Exception as e: - logger.error("%s caused Exception: %s", item.etag, e.message) + logger.error("{0} caused Exception: {1}".format(item.etag, e)) return None From c792a802ce0d1bd615239459b63f324db0d4456e Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 13 Jan 2021 19:13:33 -0500 Subject: [PATCH 021/175] StandardError exception name was removed in python3 Also a comment cleanup (item already fixed). --- idigbio_ingestion/mediaing/fetcher.py | 4 ++-- idigbio_workers/lib/download.py | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/idigbio_ingestion/mediaing/fetcher.py b/idigbio_ingestion/mediaing/fetcher.py index e12015f5..e7c66442 100644 --- a/idigbio_ingestion/mediaing/fetcher.py +++ b/idigbio_ingestion/mediaing/fetcher.py @@ -274,7 +274,7 @@ def get_media(self): if self.ok: logger.info("Success! %s", self.url) return self - except StandardError: + except: self.status_code = Status.UNHANDLED_FAILURE logger.exception("Unhandled error processing: %s", self.url) return self @@ -450,7 +450,7 @@ def get_media(self): self.url, self.sleep_retry, self.retries) sleep(self.sleep_retry) self.sleep_retry *= 1.5 - except StandardError: + except: self.status_code = Status.UNHANDLED_FAILURE logger.exception("Unhandled %s", self.url) return self diff --git a/idigbio_workers/lib/download.py b/idigbio_workers/lib/download.py index 1c8e66db..efd88a1a 100644 --- a/idigbio_workers/lib/download.py +++ b/idigbio_workers/lib/download.py @@ -276,9 +276,6 @@ def make_file(t, query, raw=False, tabs=False, fields=None, es = get_connection() mapping = es.indices.get_mapping(index=indexName, doc_type=t) - #FIXME - # Can't get the zeroeth element out of a dictionary anymore. - # https://stackoverflow.com/questions/17431638/get-typeerror-dict-values-object-does-not-support-indexing-when-using-python mapping_root = mapping.values()[0]["mappings"][t]["properties"] if raw: mapping_root = mapping_root["data"]["properties"] From 2afe845ea5f8e739bf5c0c1ad8fd841e718eb897 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 13 Jan 2021 19:15:59 -0500 Subject: [PATCH 022/175] imap replaced with map, all strings in python3 are the same type --- idigbio_ingestion/mediaing/fetcher.py | 2 +- idigbio_workers/lib/download.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/idigbio_ingestion/mediaing/fetcher.py b/idigbio_ingestion/mediaing/fetcher.py index e7c66442..0407259b 100644 --- a/idigbio_ingestion/mediaing/fetcher.py +++ b/idigbio_ingestion/mediaing/fetcher.py @@ -113,7 +113,7 @@ def process_list(fetchitems, forprefix=''): uploadpool = gevent.pool.Pool(8) items = fetchrpool.imap_unordered(lambda fi: fi.get_media(), fetchitems, maxsize=10) items = uploadpool.imap_unordered(lambda fi: fi.upload_to_storage(store), items, maxsize=10) - items = itertools.imap(FetchItem.cleanup, items) + items = map(FetchItem.cleanup, items) items = update_db_status(items) items = count_result_types(items, forprefix=forprefix) return ilen(items) # consume the generator diff --git a/idigbio_workers/lib/download.py b/idigbio_workers/lib/download.py index efd88a1a..388dcaaf 100644 --- a/idigbio_workers/lib/download.py +++ b/idigbio_workers/lib/download.py @@ -209,7 +209,7 @@ def query_to_csv(outf, t, body, header_fields, fields, id_field, raw, tabs, id_f for k in fields: v = get_source_value(r["_source"],k) if v is not None: - if isinstance(v, str) or isinstance(v, unicode): + if isinstance(v, str): r_fields.append(v) else: r_fields.append(json.dumps(v)) From ebc37cec70e1c88f04723108dc0854f9e35658af Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 13 Jan 2021 19:16:42 -0500 Subject: [PATCH 023/175] Exception no longer has a "message" in python3 --- idigbio_ingestion/mediaing/fetcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idigbio_ingestion/mediaing/fetcher.py b/idigbio_ingestion/mediaing/fetcher.py index 0407259b..cc85c839 100644 --- a/idigbio_ingestion/mediaing/fetcher.py +++ b/idigbio_ingestion/mediaing/fetcher.py @@ -288,7 +288,7 @@ def fetch(self): self.reason = str(mii) self.status_code = Status.UNREQUESTABLE except ConnectionError as connectione: - self.reason = "{0} {1}".format(connectione.errno, connectione.message) + self.reason = str(connectione.errno) + "{0}".format(connectione) self.status_code = Status.CONNECTION_ERROR else: try: From 95f9abb0a4b115c2a636dfcca8d5c2ca4026bdb4 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 13 Jan 2021 19:35:35 -0500 Subject: [PATCH 024/175] put comment back because not fixed yet --- idigbio_workers/lib/download.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/idigbio_workers/lib/download.py b/idigbio_workers/lib/download.py index 388dcaaf..46d32e41 100644 --- a/idigbio_workers/lib/download.py +++ b/idigbio_workers/lib/download.py @@ -276,6 +276,9 @@ def make_file(t, query, raw=False, tabs=False, fields=None, es = get_connection() mapping = es.indices.get_mapping(index=indexName, doc_type=t) + #FIXME + # Can't get the zeroeth element out of a dictionary anymore. + # https://stackoverflow.com/questions/17431638/get-typeerror-dict-values-object-does-not-support-indexing-when-using-python mapping_root = mapping.values()[0]["mappings"][t]["properties"] if raw: mapping_root = mapping_root["data"]["properties"] From 7004efffc479a8450f2d774562158201515f3806 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 13 Jan 2021 19:36:11 -0500 Subject: [PATCH 025/175] "reload" moved to importlib in python3 --- tests/idb/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/idb/conftest.py b/tests/idb/conftest.py index f4f7f0d0..07ca2b7e 100644 --- a/tests/idb/conftest.py +++ b/tests/idb/conftest.py @@ -2,6 +2,7 @@ import gevent import os import sys +from importlib import reload @pytest.fixture def app(testdbpool, testdata, logger): From d7e0304c11e485f23be19741753dd81c079bf3ef Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 13 Jan 2021 19:39:49 -0500 Subject: [PATCH 026/175] main3? Only fixing the print statements for now --- idigbio_ingestion/db_rsids.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/idigbio_ingestion/db_rsids.py b/idigbio_ingestion/db_rsids.py index ac62beb6..93c4c30c 100644 --- a/idigbio_ingestion/db_rsids.py +++ b/idigbio_ingestion/db_rsids.py @@ -38,7 +38,7 @@ def main3(): def main2(): for r in PostgresDB().get_type_list("recordset", limit=None): try: - print r["uuid"] + print (r["uuid"]) except: traceback.print_exc() @@ -48,7 +48,7 @@ def main1(): o = r.json() for rs in o["items"]: - print rs["uuid"] + print (rs["uuid"]) if __name__ == '__main__': main3() From 1ff7ed2908ab007499474f668c566bef72607320 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 13 Jan 2021 19:58:58 -0500 Subject: [PATCH 027/175] json.dump requires the file-like object to be string-writable From https://docs.python.org/3/library/json.html The json module always produces str objects, not bytes objects. Therefore, fp.write() must support str input. --- idigbio_ingestion/db_check.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/idigbio_ingestion/db_check.py b/idigbio_ingestion/db_check.py index ed7f93d8..a38f05ed 100644 --- a/idigbio_ingestion/db_check.py +++ b/idigbio_ingestion/db_check.py @@ -515,7 +515,7 @@ def process_file(fname, mime, rsid, existing_etags, existing_ids, ingest=False, def save_summary_json(rsid, counts): - with AtomicFile(rsid + ".summary.json", "wb") as sumf: + with AtomicFile(rsid + ".summary.json", "w") as sumf: json.dump(counts, sumf, indent=2) @@ -542,7 +542,7 @@ def metadataToSummaryJSON(rsid, metadata, writeFile=True, doStats=True): if metadata["filemd5"] is None: summary["datafile_ok"] = False if writeFile: - with AtomicFile(rsid + ".summary.json", "wb") as jf: + with AtomicFile(rsid + ".summary.json", "w") as jf: json.dump(summary, jf, indent=2) return summary @@ -573,9 +573,9 @@ def metadataToSummaryJSON(rsid, metadata, writeFile=True, doStats=True): stats.index(doc_type='digest', body=summary) if writeFile: - with AtomicFile(rsid + ".summary.json", "wb") as jf: + with AtomicFile(rsid + ".summary.json", "w") as jf: json.dump(summary, jf, indent=2) - with AtomicFile(rsid + ".metadata.json", "wb") as jf: + with AtomicFile(rsid + ".metadata.json", "w") as jf: json.dump(metadata, jf, indent=2) else: return summary @@ -638,9 +638,9 @@ def main(rsid, ingest=False): else: rlogger.info("Building ids/uuids json") db_u_d, db_i_d = get_db_dicts(rsid) - with AtomicFile(rsid + "_uuids.json", "wb") as uuidf: + with AtomicFile(rsid + "_uuids.json", "w") as uuidf: json.dump(db_u_d, uuidf) - with AtomicFile(rsid + "_ids.json", "wb") as idf: + with AtomicFile(rsid + "_ids.json", "w") as idf: json.dump(db_i_d, idf) commit_force = False From 90a7561a12dd14fc2675ae44d0bb41541ea128c3 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 13 Jan 2021 20:11:37 -0500 Subject: [PATCH 028/175] all strings are the same type in python3 --- idb/helpers/conversions.py | 4 ++-- idb/helpers/etags.py | 2 +- idigbio_ingestion/lib/delimited.py | 2 +- idigbio_workers/lib/query_shim.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/idb/helpers/conversions.py b/idb/helpers/conversions.py index 47215b51..4336bcb5 100644 --- a/idb/helpers/conversions.py +++ b/idb/helpers/conversions.py @@ -277,7 +277,7 @@ def getfield(f, d, t="text"): if t == "list": return [x.lower().strip() for x in d[f]] else: - if isinstance(d[f], str) or isinstance(d[f], unicode): + if isinstance(d[f], str) or isinstance(d[f], str): return d[f].lower().strip() else: return d[f] @@ -375,7 +375,7 @@ def intGrabber(t, d): for f in ef[t]: fv = getfield(f[1], d) if fv is not None: - if isinstance(fv, (str, unicode)): + if isinstance(fv, str): try: n = grabFirstNumber(fv) if n is not None: diff --git a/idb/helpers/etags.py b/idb/helpers/etags.py index 9e49ce27..97fae58e 100644 --- a/idb/helpers/etags.py +++ b/idb/helpers/etags.py @@ -52,7 +52,7 @@ def objectHasher(hash_type, data, sort_arrays=False, sort_keys=True): sa.sort() s = "".join(sa) - elif isinstance(data, str) or isinstance(data, unicode): + elif isinstance(data, str) or isinstance(data, str): s = data elif isinstance(data, int) or isinstance(data, float): s = str(data) diff --git a/idigbio_ingestion/lib/delimited.py b/idigbio_ingestion/lib/delimited.py index 981b0611..54559f39 100644 --- a/idigbio_ingestion/lib/delimited.py +++ b/idigbio_ingestion/lib/delimited.py @@ -57,7 +57,7 @@ def __init__(self, fh, encoding="utf8", delimiter=",", fieldenc="\"", header=Non self.lineCount = 0 self.lineLength = None - if isinstance(fh, str) or isinstance(fh, unicode): + if isinstance(fh, str): self.name = fh else: self.name = fh.name diff --git a/idigbio_workers/lib/query_shim.py b/idigbio_workers/lib/query_shim.py index e77e224a..3bfc6a3d 100644 --- a/idigbio_workers/lib/query_shim.py +++ b/idigbio_workers/lib/query_shim.py @@ -17,7 +17,7 @@ class TermNotFoundExcpetion(Exception): def isString(v): - return isinstance(v, str) or isinstance(v, unicode) + return isinstance(v, str) def existsFilter(k): From f66612759e4024133d886f6dd38ce6e7def3f8c1 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 13 Jan 2021 21:02:50 -0500 Subject: [PATCH 029/175] no longer write strings in binary mode --- idigbio_workers/lib/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idigbio_workers/lib/download.py b/idigbio_workers/lib/download.py index 46d32e41..b25c71bf 100644 --- a/idigbio_workers/lib/download.py +++ b/idigbio_workers/lib/download.py @@ -56,7 +56,7 @@ def write_citation_file(dl_id, t, query, recordsets): now = datetime.datetime.now() rs_string = "\n".join(rs_strings) + "\n" - with AtomicFile(filename, "wb") as citefile: + with AtomicFile(filename, "w") as citefile: # 0: Current Year # 1: Query Text # 2: Total Number of Records From e149540876e8c1c413cbccd6c1e8231ba241add8 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 13 Jan 2021 21:03:37 -0500 Subject: [PATCH 030/175] Stop trying to get indexed item out of unordered dict Grab the "next" aka "first" item from the dictionary instead since we cannot get a zeroeth element out of a dictionary. https://stackoverflow.com/questions/17431638/get-typeerror-dict-values-object-does-not-support-indexing-when-using-python --- idigbio_workers/lib/download.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/idigbio_workers/lib/download.py b/idigbio_workers/lib/download.py index b25c71bf..36554587 100644 --- a/idigbio_workers/lib/download.py +++ b/idigbio_workers/lib/download.py @@ -276,10 +276,7 @@ def make_file(t, query, raw=False, tabs=False, fields=None, es = get_connection() mapping = es.indices.get_mapping(index=indexName, doc_type=t) - #FIXME - # Can't get the zeroeth element out of a dictionary anymore. - # https://stackoverflow.com/questions/17431638/get-typeerror-dict-values-object-does-not-support-indexing-when-using-python - mapping_root = mapping.values()[0]["mappings"][t]["properties"] + mapping_root = next(iter(mapping.values()))["mappings"][t]["properties"] if raw: mapping_root = mapping_root["data"]["properties"] From b7595d5b848fffad13f3e36fdb4ada8bb6d83cab Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 13 Jan 2021 21:07:03 -0500 Subject: [PATCH 031/175] convert boto md5 bytes to string for comparison --- tests/idb/test_helpers_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/idb/test_helpers_storage.py b/tests/idb/test_helpers_storage.py index 89351f47..cd203573 100644 --- a/tests/idb/test_helpers_storage.py +++ b/tests/idb/test_helpers_storage.py @@ -48,7 +48,7 @@ def test_download_md5_validation(store, existingkey, tmpdir): def test_file_upload_download(store, bucketname, tmpdir): k = store.upload(store.get_key('foobar', bucketname), __file__, content_type="x-foo/bar", public=False) localmd5 = calcFileHash(__file__) - assert k.md5 == localmd5 + assert k.md5.decode('utf-8') == localmd5 k2 = store.get_key('foobar', bucketname) assert k2.exists() From ecad8bdf2eb5a18bb8d6c45e9f3c8838fc65b3c9 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 13 Jan 2021 21:25:51 -0500 Subject: [PATCH 032/175] itertools izip_longest was replaced with zip_longest in python3 --- idb/helpers/__init__.py | 2 +- idb/postgres_backend/db.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/idb/helpers/__init__.py b/idb/helpers/__init__.py index 9c57e996..76ace50c 100644 --- a/idb/helpers/__init__.py +++ b/idb/helpers/__init__.py @@ -28,4 +28,4 @@ def grouper(iterable, n, fillvalue=None): """ # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx args = [iter(iterable)] * n - return itertools.izip_longest(fillvalue=fillvalue, *args) + return itertools.zip_longest(fillvalue=fillvalue, *args) diff --git a/idb/postgres_backend/db.py b/idb/postgres_backend/db.py index 5de90596..d7f3c545 100644 --- a/idb/postgres_backend/db.py +++ b/idb/postgres_backend/db.py @@ -588,7 +588,7 @@ class MediaObject(object): ) def __init__(self, *args, **kwargs): - for s, a in itertools.izip_longest(self.__slots__, args): + for s, a in itertools.zip_longest(self.__slots__, args): setattr(self, s, a) for kw in kwargs.items(): setattr(self, *kw) From f876577b9c6afda8d40eb1344245e4af51b223fa Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 17 Jan 2021 17:00:59 -0500 Subject: [PATCH 033/175] update README with python 3 instructions --- README.md | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index eb07855e..ed5c82f2 100644 --- a/README.md +++ b/README.md @@ -8,27 +8,23 @@ iDigBio server and backend code for data ingestion and data API. ### System Dependencies -Currently this project only works in python2.7 and is not compatible with Python 3. - -The following library packages will need to be installed to run the api: - -In Ubuntu 16.04: - - apt-get install python2.7-dev libblas-dev liblapack-dev \ - libatlas-base-dev gfortran libgdal-dev libpq-dev libgeos-c1v5 \ - libsystemd-dev +#### Python 3 instructions: In Ubuntu 18.04: - apt install python-dev libblas-dev liblapack-dev \ + apt install python3-dev libblas-dev liblapack-dev \ libatlas-base-dev gfortran libgdal-dev libpq-dev libgeos-c1v5 \ libsystemd-dev For Ingestion and Development, the following are also needed: -In Ubuntu 16.04, 18.04: +In 18.04: + + apt install libxml2 libxslt1-dev ffmpeg fonts-dejavu-core libfreetype6-dev python-systemd - apt-get install libxml2 libxslt1-dev ffmpeg fonts-dejavu-core libfreetype6-dev python-systemd +#### Python 2.7 instructions: + +This version of idb-backend will not work under Python 2. ### Package installation @@ -91,6 +87,8 @@ celery to a background worker. You probably want to run in a virtual environment. You may wish to disable the pip cache to verify package builds are working properly. +*in progress for Python 3...* + ```bash $ virtualenv -p python2.7 .venv From dc4d24a4639f57e819c4d08aeb10419189100c3e Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 18 Jan 2021 18:45:57 -0500 Subject: [PATCH 034/175] izip_longest renamed to zip_longest in python3 --- idb/helpers/__init__.py | 2 +- idb/postgres_backend/db.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/idb/helpers/__init__.py b/idb/helpers/__init__.py index 9c57e996..76ace50c 100644 --- a/idb/helpers/__init__.py +++ b/idb/helpers/__init__.py @@ -28,4 +28,4 @@ def grouper(iterable, n, fillvalue=None): """ # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx args = [iter(iterable)] * n - return itertools.izip_longest(fillvalue=fillvalue, *args) + return itertools.zip_longest(fillvalue=fillvalue, *args) diff --git a/idb/postgres_backend/db.py b/idb/postgres_backend/db.py index 5de90596..d7f3c545 100644 --- a/idb/postgres_backend/db.py +++ b/idb/postgres_backend/db.py @@ -588,7 +588,7 @@ class MediaObject(object): ) def __init__(self, *args, **kwargs): - for s, a in itertools.izip_longest(self.__slots__, args): + for s, a in itertools.zip_longest(self.__slots__, args): setattr(self, s, a) for kw in kwargs.items(): setattr(self, *kw) From b1a51b1fc5a991629391fef660f2115b3b6b1dde Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 18 Jan 2021 18:54:52 -0500 Subject: [PATCH 035/175] convert bytes to string for comparison --- tests/idb/test_data_api_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/idb/test_data_api_basic.py b/tests/idb/test_data_api_basic.py index fc3ff087..414c150d 100644 --- a/tests/idb/test_data_api_basic.py +++ b/tests/idb/test_data_api_basic.py @@ -17,7 +17,7 @@ def test_app_db(client): def test_version(client): r = client.get(url_for('version')) assert r.status_code == 200 - assert r.data == idb.__version__ + assert str(r.data, "utf-8") == idb.__version__ @pytest.mark.readonly From 794272aa08654ae0f58334105ca938379e64b42f Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 18 Jan 2021 19:00:34 -0500 Subject: [PATCH 036/175] gevent.wsgi has been renamed to gevent.pywsgi --- idb/data_api/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idb/data_api/__init__.py b/idb/data_api/__init__.py index 4355fbf2..a09e1296 100644 --- a/idb/data_api/__init__.py +++ b/idb/data_api/__init__.py @@ -85,7 +85,7 @@ def run_server(info, host, port, reload, debugger, eager_loading, debug, wsgi): elif wsgi == 'gevent': from gevent.pool import Pool - from gevent.wsgi import WSGIServer + from gevent.pywsgi import WSGIServer from idb.helpers.logging import idblogger from requestlogger import WSGILogger, ApacheFormatter logger = idblogger.getChild('api') From c8d51d4ec515d2eca37cd004f662f950ccc12bbc Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 18 Jan 2021 20:41:25 -0500 Subject: [PATCH 037/175] fixup section on Testing Dependencies --- README.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ed5c82f2..93b1856c 100644 --- a/README.md +++ b/README.md @@ -106,13 +106,21 @@ It is possible in the future that this project will be runnable using "Open in c ### Testing Dependencies -Some idb-backend tests depend on external resources, such as a database or Elasticsearch. +Some idb-backend tests depend on external resources, such as a local test postgres database or the production Elasticsearch cluster. -A local postgresql DB named `test_idigbio` with user/pass `test` / `test` must exist for many of the tests to run. Note: The data in the DB will be destroyed during the testing. +* Database tests will be SKIPPED if the local postgres test database is not available. -Database tests will be SKIPPED if the database is not available. +* Tests that depend on Elasticsearch will FAIL if the Elasticsearch cluster cannot be reached (fail very slowly in fact), or if there is some other failure. -Tests that depend on Elasticsearch will FAIL if the computer running the tests cannot reach the Elasticsearch cluster (fail very slowly in fact), or if there is some other failure. +The local postgresql 9.5 DB is named `test_idigbio` with user/pass `test` / `test`. + +Note: The data in the db with that name will be destroyed during testing. + +A temporary instance of postgres running in docker will suffice: + +``` +$ docker run --rm --name postgres_test_idigbio --network host -e POSTGRES_PASSWORD=test -e POSTGRES_USER=test -e POSTGRES_DB=test_idigbio -d postgres:9.5 +``` ### Running tests From 44c9724174e56b50e09c1a32dd47d1a386979f44 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 19 Jan 2021 10:01:11 -0500 Subject: [PATCH 038/175] bump major version of idb-backend for Python 3 migration We will not provide dual support with Python 2, so this major version change will indicate the compatibility break for the move to Python 3. --- idb/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/idb/__init__.py b/idb/__init__.py index c11769ec..4349a30d 100644 --- a/idb/__init__.py +++ b/idb/__init__.py @@ -1 +1,2 @@ -__version__ = "3.0.7" +# This sets the version for the generated idb-backend PyPi package also. +__version__ = "4.0.0" From 43721f20a93f986f490dbd672d5ce8599fdbd746 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 19 Jan 2021 10:06:04 -0500 Subject: [PATCH 039/175] setup.py changes for Python 3, update Flask --- setup.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 047bd34a..7282ec14 100644 --- a/setup.py +++ b/setup.py @@ -15,10 +15,11 @@ def read(*paths): version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', read(os.path.join(os.path.dirname(__file__), 'idb/__init__.py')), re.MULTILINE).group(1) -# Abort if on Python 3 since we know the codebase does not support it. +# Abort if not on Python 3 since we know the codebase does not support it. if sys.version_info < (3,0): sys.exit("idb-backend: Python 2 is not supported in this branch.") +# TODO: revisit this after migrate to Python 3 # Pillow-SIMD causes segfault on newer Python 2. # We do not yet support Python 3. # Only install Pillow-SIMD where it is known to work, otherwise @@ -51,8 +52,9 @@ def read(*paths): 'pytz>=2016.10', 'requests==2.20.0', 'urllib3<1.25,>=1.21.1', - 'pycrypto', - 'flask>=0.11.0, <1.0.0', + 'pycrypto', + #'flask>=0.11.0, <1.0.0', + 'flask==1.1.2', 'Flask-UUID', 'Flask-CORS', 'coverage', @@ -101,7 +103,7 @@ def read(*paths): 'License :: OSI Approved :: MIT License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', ], entry_points=''' [console_scripts] From 47f3c822733c7f8620ff55828ab5814b4bb732e8 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 19 Jan 2021 10:33:10 -0500 Subject: [PATCH 040/175] only need path from os here --- setup.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 7282ec14..1a81683f 100644 --- a/setup.py +++ b/setup.py @@ -1,19 +1,17 @@ -import os +from os import path import re import sys from setuptools import setup, find_packages -from codecs import open - def read(*paths): """Build a file path from *paths* and return the contents.""" - with open(os.path.join(*paths), 'r', 'utf-8') as f: + with open(path.join(*paths), 'r', encoding='utf-8') as f: return f.read() readme = read('README.md') version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', - read(os.path.join(os.path.dirname(__file__), 'idb/__init__.py')), re.MULTILINE).group(1) + read(path.join(path.dirname(__file__), 'idb/__init__.py')), re.MULTILINE).group(1) # Abort if not on Python 3 since we know the codebase does not support it. if sys.version_info < (3,0): From 2e172ef4ad266c5b4ccff73335cb83b0e422df47 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 19 Jan 2021 11:22:55 -0500 Subject: [PATCH 041/175] test_data_api_downloads python3 compatibility * use udatetime pure python implementation from_string instead * fake redis client returns keys that are bytes --- tests/idb/test_data_api_downloads.py | 36 ++++++++++++++++------------ 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/tests/idb/test_data_api_downloads.py b/tests/idb/test_data_api_downloads.py index 42910235..b2021327 100644 --- a/tests/idb/test_data_api_downloads.py +++ b/tests/idb/test_data_api_downloads.py @@ -6,7 +6,8 @@ import gevent import udatetime -from udatetime.rfc3339 import from_rfc3339_string +from udatetime import from_string + import pytest from flask import url_for @@ -75,7 +76,11 @@ def setupredis(client, tid=None, task_status=None, download_url=None, error=None redisdata['download_url'] = download_url if error: redisdata['error'] = error - client.hmset(rtkey, redisdata) + # hmset creates redis a hash key b'downloader:ab219ed4-78fc-4433-8ea1-c350fb47b645' + # that contains all of the key value pairs from redisdata as bytes such as: + # b'created':b'2021-01-19T14:14:46.528067+00:00', + # b'task_status':b'SUCCESS' + client.hmset(rtkey, redisdata) client.expire(rtkey, timedelta(hours=1)) if link: rqhk = DOWNLOADER_TASK_PREFIX + redisdata['hash'] @@ -91,8 +96,8 @@ def test_status_complete_task(client, fakeredcli): assert resp.json['download_url'] == "http://example.com" assert resp.json['complete'] is True assert resp.json['task_status'] == 'SUCCESS' - assert from_rfc3339_string(resp.json['created']) - assert from_rfc3339_string(resp.json['expires']) + assert from_string(resp.json['created']) + assert from_string(resp.json['expires']) assert 'error' not in resp.json @@ -103,8 +108,8 @@ def test_status_pending_task(client, fakeredcli, fakeresult): assert resp.status_code == 200 assert resp.json['task_status'] == "PENDING" assert resp.json['complete'] is False - assert from_rfc3339_string(resp.json['created']) - assert from_rfc3339_string(resp.json['expires']) + assert from_string(resp.json['created']) + assert from_string(resp.json['expires']) assert 'download_url' not in resp.json assert 'error' not in resp.json @@ -118,8 +123,8 @@ def test_status_pending_now_success(client, fakeredcli, fakeresult): assert resp.json['download_url'] == "http://foo/bar" assert resp.json['complete'] is True assert resp.json['task_status'] == 'SUCCESS' - assert from_rfc3339_string(resp.json['created']) - assert from_rfc3339_string(resp.json['expires']) + assert from_string(resp.json['created']) + assert from_string(resp.json['expires']) assert 'error' not in resp.json @@ -132,8 +137,8 @@ def test_status_pending_now_failure(client, fakeredcli, fakeresult): assert resp.json['error'] == "woot" assert resp.json['complete'] is True assert resp.json['task_status'] == 'FAILURE' - assert from_rfc3339_string(resp.json['created']) - assert from_rfc3339_string(resp.json['expires']) + assert from_string(resp.json['created']) + assert from_string(resp.json['expires']) assert 'download_url' not in resp.json gevent.wait() assert tid != fakeredcli.get(DOWNLOADER_TASK_PREFIX + resp.json['hash']), \ @@ -150,8 +155,8 @@ def test_status_pending_now_failure_already_unlinked(client, fakeredcli, fakeres assert resp.json['error'] == "woot" assert resp.json['complete'] is True assert resp.json['task_status'] == 'FAILURE' - assert from_rfc3339_string(resp.json['created']) - assert from_rfc3339_string(resp.json['expires']) + assert from_string(resp.json['created']) + assert from_string(resp.json['expires']) assert 'download_url' not in resp.json assert "woot" == fakeredcli.get(DOWNLOADER_TASK_PREFIX + "foobar") @@ -167,11 +172,12 @@ def test_initialization(client, fakeredcli, fakeresult): assert 'status_url' in resp.json tid = resp.json['status_url'].split('/')[-1] data = fakeredcli.hgetall(DOWNLOADER_TASK_PREFIX + tid) + # At this point, data contains keys that are bytes aka data[b'query'] assert data assert len(data) != 0 - assert data['query'] - assert data['hash'] - assert data['created'] + assert data[b'query'] + assert data[b'hash'] + assert data[b'created'] def test_initialization_repeated_request(client, fakeredcli, fakeresult): From 20762f2f4def2c6fa36789b9b24884e83d5d2a92 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 1 Feb 2021 19:56:57 -0500 Subject: [PATCH 042/175] tell redis_client to decode responses In python3, redis returns bytes for "everything" by default (both keynames and data). Adding decode_responses=True should force each field name and contents to be a string (instead of bytes) so our string-based comparisons continue to work. e.g. redis_response["status_url"] vs redis_response[b"status_url"] Setting decode_responses fixed the test suite but leaving this comment here as a marker in case production behavior differs. --- idb/data_api/v2_download.py | 2 ++ idigbio_workers/__init__.py | 8 +++++++- tests/idb/test_data_api_downloads.py | 9 +++++---- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/idb/data_api/v2_download.py b/idb/data_api/v2_download.py index 4dcac90a..73816d9c 100644 --- a/idb/data_api/v2_download.py +++ b/idb/data_api/v2_download.py @@ -107,6 +107,8 @@ def get_task_status(tid): rconn = get_redis_conn() rtkey = DOWNLOADER_TASK_PREFIX + tid try: + # In python3, redis returns bytes by default for both keynames and data. + # See important change in get_redis_conn(). tdata = rconn.hgetall(rtkey) if len(tdata) == 0: return None diff --git a/idigbio_workers/__init__.py b/idigbio_workers/__init__.py index 16b5d3d2..d27ae6c0 100644 --- a/idigbio_workers/__init__.py +++ b/idigbio_workers/__init__.py @@ -26,7 +26,13 @@ def get_redis_connection_params(): def get_redis_conn(): import redis - return redis.StrictRedis(**get_redis_connection_params()) + # In python3, redis returns bytes for "everything" by default (both keynames and data). + # Adding decode_responses=True should force each field name and contents to be a + # string (instead of bytes) so our string-based comparisons continue to work. + # e.g. redis_response["status_url"] vs redis_response[b"status_url"] + # Setting decode_responses fixed the test suite but leaving this comment here + # as a marker in case production behavior differs. + return redis.StrictRedis(**get_redis_connection_params(), decode_responses=True) @app.task() diff --git a/tests/idb/test_data_api_downloads.py b/tests/idb/test_data_api_downloads.py index b2021327..a05824a9 100644 --- a/tests/idb/test_data_api_downloads.py +++ b/tests/idb/test_data_api_downloads.py @@ -22,7 +22,7 @@ @pytest.fixture() def fakeredcli(request): import fakeredis - fsr = fakeredis.FakeStrictRedis() + fsr = fakeredis.FakeStrictRedis(decode_responses=True) v2_download.get_redis_conn = lambda: fsr request.addfinalizer(fsr.flushall) return fsr @@ -91,6 +91,7 @@ def setupredis(client, tid=None, task_status=None, download_url=None, error=None def test_status_complete_task(client, fakeredcli): tid = setupredis(fakeredcli, task_status="SUCCESS", download_url="http://example.com") + # client is Flask client! resp = client.get(url_for('idb.data_api.v2_download.status', tid=tid)) assert resp.status_code == 200 assert resp.json['download_url'] == "http://example.com" @@ -175,9 +176,9 @@ def test_initialization(client, fakeredcli, fakeresult): # At this point, data contains keys that are bytes aka data[b'query'] assert data assert len(data) != 0 - assert data[b'query'] - assert data[b'hash'] - assert data[b'created'] + assert data['query'] + assert data['hash'] + assert data['created'] def test_initialization_repeated_request(client, fakeredcli, fakeresult): From b0604f43a9939494f462cea222eeb7fe83dfece6 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 3 Feb 2021 08:36:02 -0500 Subject: [PATCH 043/175] Let the library open and read bytes of XML from file We've been doing it wrong. Rather than reading "text" into a buffer and then trying to parse the buffer, let the library handle the reading of bytes also. See https://lxml.de/FAQ.html#why-can-t-lxml-parse-my-xml-from-unicode-strings --- idigbio_ingestion/lib/eml.py | 13 ++++--------- tests/idigbio_ingestion/lib/test_eml.py | 4 ++-- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/idigbio_ingestion/lib/eml.py b/idigbio_ingestion/lib/eml.py index dcb30b8d..3bcb34e3 100644 --- a/idigbio_ingestion/lib/eml.py +++ b/idigbio_ingestion/lib/eml.py @@ -9,12 +9,8 @@ def getElement(root,name): return root.find(name) -def parseEml(id, emlText): - "Returns a dictionary of the supplied emlText" - - # dump the full eml/xml for debugging - #logger.debug(emlText) - +def parseEml(id, emlFilename): + "Returns a dictionary of fields from the eml file" # If the target eml document is not XML, the eml object will not be created due to XMLSyntaxError or other # pyquery exception. This is known to occur when a link to eml results in a 404 error page containing HTML. @@ -22,7 +18,7 @@ def parseEml(id, emlText): # It is possible we could trap this ahead of time by checking the raw emlText for key xml features # or HTML document features. - eml = pq(emlText, parser='xml') + eml = pq(filename=emlFilename, parser='xml') ### The eml().txt() function returns an empty string instead of None if the location does not exist in the eml ### (if there is "no text node" according to release notes https://pypi.python.org/pypi/pyquery) @@ -155,8 +151,7 @@ def parseEml(id, emlText): def main(): import sys import json - with open(sys.argv[1],"rb") as inf: - print (json.dumps(parseEml("testid",inf.read()))) + print (json.dumps(parseEml("testid",sys.argv[1]))) if __name__ == '__main__': diff --git a/tests/idigbio_ingestion/lib/test_eml.py b/tests/idigbio_ingestion/lib/test_eml.py index 21d5ac46..65b8c836 100644 --- a/tests/idigbio_ingestion/lib/test_eml.py +++ b/tests/idigbio_ingestion/lib/test_eml.py @@ -29,6 +29,6 @@ def test_intellectual_rights(eml_filename, expected_license, emlpathdir): - emlfile = emlpathdir.join(eml_filename).open() - parsed_eml = parseEml('id_placeholder_test_suite', emlfile.read()) + emlfilename = emlpathdir.join(eml_filename) + parsed_eml = parseEml('id_placeholder_test_suite', emlfilename) assert parsed_eml['data_rights'] == expected_license From fb44f4b12b650041c85158616d037df5d2c384e1 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 3 Feb 2021 08:59:37 -0500 Subject: [PATCH 044/175] iterables fixes --- idb/helpers/conversions.py | 4 ++-- idb/helpers/etags.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/idb/helpers/conversions.py b/idb/helpers/conversions.py index 4336bcb5..677ac320 100644 --- a/idb/helpers/conversions.py +++ b/idb/helpers/conversions.py @@ -936,11 +936,11 @@ def grabAll(t, d): # r.update(geoshape_fill(t, d, r)) r["flags"] = setFlags(r) - for k in r.keys(): + for k in list(r): if k.startswith("flag_"): r["flags"].append("_".join(k.split("_")[1:])) del r[k] - for k in d.keys(): + for k in list(d): if k.startswith("flag_"): r["flags"].append("_".join(k.split("_")[1:])) r["dqs"] = score(t, r) diff --git a/idb/helpers/etags.py b/idb/helpers/etags.py index 97fae58e..d0bf217e 100644 --- a/idb/helpers/etags.py +++ b/idb/helpers/etags.py @@ -57,7 +57,7 @@ def objectHasher(hash_type, data, sort_arrays=False, sort_keys=True): elif isinstance(data, int) or isinstance(data, float): s = str(data) elif isinstance(data, dict): - ks = data.keys() + ks = list(data.keys()) if sort_keys: ks.sort() From 7950a2f1c5486db099366525dc29dd33acba8ccb Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 3 Feb 2021 09:30:51 -0500 Subject: [PATCH 045/175] iteritems() renamed to items() in python3 --- idb/data_api/v2_media.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idb/data_api/v2_media.py b/idb/data_api/v2_media.py index b2fec488..3e4cdbc3 100644 --- a/idb/data_api/v2_media.py +++ b/idb/data_api/v2_media.py @@ -193,7 +193,7 @@ def upload(): j = request.get_json() if j is not None: vals.update(j) - for k, v in request.values.iteritems(): + for k, v in request.values.items(): vals[k] = v filereference = vals.get("filereference") From 9d1dc69cd78b4e472611d2f49c40ecc364eeb73f Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 3 Feb 2021 09:31:30 -0500 Subject: [PATCH 046/175] b64encode needs a bytes-like object --- tests/idb/test_data_api_media.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/idb/test_data_api_media.py b/tests/idb/test_data_api_media.py index 129e0d8d..0d4f593a 100644 --- a/tests/idb/test_data_api_media.py +++ b/tests/idb/test_data_api_media.py @@ -18,7 +18,8 @@ def valid_auth_header(mocker): mocker.patch.object(idb_flask_authn, 'check_auth', return_value=True) uuid = "872733a2-67a3-4c54-aa76-862735a5f334" key = "3846c98586668822ba6d5cb69caeb4c6" - return ('Authorization', 'Basic ' + base64.b64encode("{}:{}".format(uuid, key))) + authstring = base64.b64encode(bytes(f"{uuid}:{key}", 'utf-8')).decode('utf-8') + return ('Authorization', 'Basic ' + authstring) @pytest.fixture() From cd36ebeb81b50e8749cd3357554f630454727c38 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Thu, 4 Feb 2021 11:47:41 -0500 Subject: [PATCH 047/175] move testing doc in main README to the existing tests/README --- README.md | 97 ++------------------------------------------- tests/README.md | 103 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 104 insertions(+), 96 deletions(-) diff --git a/README.md b/README.md index 93b1856c..5034e66e 100644 --- a/README.md +++ b/README.md @@ -104,104 +104,13 @@ $ pip --no-cache-dir install -r requirements.txt It is possible in the future that this project will be runnable using "Open in container" features of Microsoft Visual Studio Code (aka vscode or just `code`). -### Testing Dependencies -Some idb-backend tests depend on external resources, such as a local test postgres database or the production Elasticsearch cluster. - -* Database tests will be SKIPPED if the local postgres test database is not available. - -* Tests that depend on Elasticsearch will FAIL if the Elasticsearch cluster cannot be reached (fail very slowly in fact), or if there is some other failure. - -The local postgresql 9.5 DB is named `test_idigbio` with user/pass `test` / `test`. - -Note: The data in the db with that name will be destroyed during testing. - -A temporary instance of postgres running in docker will suffice: - -``` -$ docker run --rm --name postgres_test_idigbio --network host -e POSTGRES_PASSWORD=test -e POSTGRES_USER=test -e POSTGRES_DB=test_idigbio -d postgres:9.5 -``` - -### Running tests +### Running the Test Suite The test suite can be run by executing `py.test` (or `pytest`). -However due to the dependencies mentioned above, you may wish to run the database in docker each time. The sleep is needed to allow postgres time to start accepting connections. - - docker run --rm --name postgres_test_idigbio --network host \ - -e POSTGRES_PASSWORD=test -e POSTGRES_USER=test -e POSTGRES_DB=test_idigbio \ - -d postgres:9.5 && \ - sleep 5; \ - py.test ; \ - docker stop postgres_test_idigbio - - -To exclude a single set of tests that are failing (or Seg Faulting!), add the `--deselect` option to the pytest command: - - py.test --deselect=tests/idigbio_ingestion/mediaing/test_derivatives.py - -To find out why tests are being Skipped, add the `-rxs` options. - -A "what the heck is going on with the tests and skip the one that is Seg Faulting" example command: - - docker run --rm --name postgres_test_idigbio --network host \ - -e POSTGRES_PASSWORD=test -e POSTGRES_USER=test -e POSTGRES_DB=test_idigbio \ - -d postgres:9.5 && \ - sleep 5; \ - py.test -rxs --deselect=tests/idigbio_ingestion/mediaing/test_derivatives.py ; \ - docker stop postgres_test_idigbio - -### Create a local postgres DB +See the [README.md in the `tests` subdirectory](tests/README.md) for more information. -The recommended approach is to run postgres via docker (see above). +There are a number of important dependencies noted there. -If you have a full installation of postgres running locally, the db can be manually created with: - - createuser -l -e test -P - createdb -l 'en_US.UTF-8' -E UTF8 -O test -e test_idigbio; - - # The schema obj is still owned by the user of the above - # statement, not the owner 'test'. Drop it so it will be recreated - # by the script appropriately - psql -c "DROP SCHEMA public CASCADE;" test_idigbio - - -### Schema - -The live production db schema is copied into `tests/data/schema.sql` by periodically running this command: - - pg_dump --host c18node8.acis.ufl.edu --username idigbio \ - --format plain --schema-only --schema=public \ - --clean --if-exists \ - --no-owner --no-privileges --no-tablespaces --no-unlogged-table-data \ - --file tests/data/schema.sql \ - idb_api_prod - -Except not yet because there are lots of differences between the existing file and one created by running that command (due to fixes for https://wiki.postgresql.org/wiki/A_Guide_to_CVE-2018-1058:_Protect_Your_Search_Path). - - -### Data - -A trimmed down set of data has been manually curated to support the test suite. It is provided in `tests/data/testdata.sql` - -The full dump / original was created with something like: - - pg_dump --port 5432 --host c18node8.acis.ufl.edu --username idigbio \ - --format plain --data-only --encoding UTF8 \ - --inserts --column-inserts --no-privileges --no-tablespaces \ - --verbose --no-unlogged-table-data \ - --exclude-table-data=ceph_server_files \ - --file tests/data/testdata.sql idb_api_prod - -Such a dump is huge and un-usable and un-editable by normal means. It is not clear how the dump was transformed / curated into its current state. - -If running the dump again, consider adding multiple `--exclude-table-data=TABLE` for some of the bigger tables that are not materially relevant to test suite such as: - -```plaintext -annotations -data -corrections -ceph_server_files -``` -We likely need to find a new way to refresh the test dataset. diff --git a/tests/README.md b/tests/README.md index 3058d1bc..70c10ba9 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,4 +1,6 @@ -## tests +# Tests + +## Layout The current convention for idb-backend is that tests will live in a separate directory tree but will mimic the codebase structure. @@ -88,4 +90,101 @@ test_data_exists.py::test_pngpath_is_usable PASSED ``` -More on pytest at https://docs.pytest.org/en/latest/usage.html \ No newline at end of file +To exclude a single set of tests that are failing (or Seg Faulting!), add the `--deselect` option to the pytest command: + + py.test --deselect=tests/idigbio_ingestion/mediaing/test_derivatives.py + +To find out why tests are being Skipped, add the `-rxs` options. + + +More on pytest at https://docs.pytest.org/en/latest/usage.html + + +## Dependencies + +Some idb-backend tests depend on external resources, such as a local test postgres database or the production Elasticsearch cluster. + +* Database tests will be SKIPPED if the local postgres test database is not available. + +* Tests that depend on Elasticsearch will FAIL if the Elasticsearch cluster cannot be reached (fail very slowly in fact), or if there is some other failure. + +The local postgresql 9.5 DB is named `test_idigbio` with user/pass `test` / `test`. + +Note: The data in the db with that name will be destroyed during testing. + +A temporary instance of postgres running in docker will suffice: + +``` +$ docker run --rm --name postgres_test_idigbio --network host -e POSTGRES_PASSWORD=test -e POSTGRES_USER=test -e POSTGRES_DB=test_idigbio -d postgres:9.5 +``` + + +## Running All Tests + +Due to the dependencies mentioned above, you may wish to run the database in docker each time. The sleep is needed to allow postgres time to start accepting connections. + + docker run --rm --name postgres_test_idigbio --network host \ + -e POSTGRES_PASSWORD=test -e POSTGRES_USER=test -e POSTGRES_DB=test_idigbio \ + -d postgres:9.5 && \ + sleep 5; \ + py.test ; \ + docker stop postgres_test_idigbio + + + +## Local database + +The recommended approach is to run postgres via docker (see above). + +### Create a local postgres DB + +However, if you have a full installation of postgres running locally that you wish to use, the db can be manually created with: + + createuser -l -e test -P + createdb -l 'en_US.UTF-8' -E UTF8 -O test -e test_idigbio; + + # The schema obj is still owned by the user of the above + # statement, not the owner 'test'. Drop it so it will be recreated + # by the script appropriately + psql -c "DROP SCHEMA public CASCADE;" test_idigbio + + +### Schema + +The live production db schema is copied into `tests/data/schema.sql` by periodically running this command: + + pg_dump --host c18node8.acis.ufl.edu --username idigbio \ + --format plain --schema-only --schema=public \ + --clean --if-exists \ + --no-owner --no-privileges --no-tablespaces --no-unlogged-table-data \ + --file tests/data/schema.sql \ + idb_api_prod + +Except not yet because there are lots of differences between the existing file and one created by running that command (due to fixes for https://wiki.postgresql.org/wiki/A_Guide_to_CVE-2018-1058:_Protect_Your_Search_Path). + + +### Data + +A trimmed down set of data has been manually curated to support the test suite. It is provided in `tests/data/testdata.sql` + +The full dump / original was created with something like: + + pg_dump --port 5432 --host c18node8.acis.ufl.edu --username idigbio \ + --format plain --data-only --encoding UTF8 \ + --inserts --column-inserts --no-privileges --no-tablespaces \ + --verbose --no-unlogged-table-data \ + --exclude-table-data=ceph_server_files \ + --file tests/data/testdata.sql idb_api_prod + +Such a dump is huge and un-usable and un-editable by normal means. It is not clear how the dump was transformed / curated into its current state. + +If running the dump again, consider adding multiple `--exclude-table-data=TABLE` for some of the bigger tables that are not materially relevant to test suite such as: + +```plaintext +annotations +data +corrections +ceph_server_files +``` + +We likely need to find a new way to refresh the test dataset. From 56c053fb557af3c0c1cad7df47fce244712660f5 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Thu, 4 Feb 2021 11:57:09 -0500 Subject: [PATCH 048/175] update developer environment virtualenv setup instructions for py3 --- README.md | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5034e66e..9903e620 100644 --- a/README.md +++ b/README.md @@ -87,21 +87,38 @@ celery to a background worker. You probably want to run in a virtual environment. You may wish to disable the pip cache to verify package builds are working properly. -*in progress for Python 3...* + +### Python 3 ```bash -$ virtualenv -p python2.7 .venv + +$ python --version +Python 2.7.17 + +$ python3 -m virtualenv -p python3 .venv +Already using interpreter /usr/bin/python3 +Using base prefix '/usr' +New python executable in /tmp/venvtest/.venv/bin/python3 +Also creating executable in /tmp/venvtest/.venv/bin/python +Installing setuptools, pkg_resources, pip, wheel...done. $ source .venv/bin/activate $ python --version -Python 2.7.17 +Python 3.6.9 $ pip --no-cache-dir install -e . $ pip --no-cache-dir install -r requirements.txt ``` +### Python 2 (obsolete) + +This project is no longer compatible with Python 2. + + +### Develop in Container + It is possible in the future that this project will be runnable using "Open in container" features of Microsoft Visual Studio Code (aka vscode or just `code`). From d0898ebf79d38d87cd2d2877e4969d75dff0732d Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Thu, 4 Feb 2021 12:09:37 -0500 Subject: [PATCH 049/175] update Dockerfile with python3 to fix build pipeline for this branch --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 959a97b1..e0ae7da6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM python:2.7 +FROM python:3.6.9 +# python:3.6.9 is based on "Debian GNU/Linux 10 (buster)" RUN apt-get update && apt-get install -y --no-install-recommends \ gfortran \ libatlas-base-dev \ From 012e81aa057dfc2381b6086920d5066f5dd38049 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Thu, 4 Feb 2021 12:17:52 -0500 Subject: [PATCH 050/175] update travis build to use python3 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 93c88867..52f5476c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ sudo: required dist: xenial language: python -python: 2.7 +python: 3.6.9 services: - docker - postgresql From b95b82263de955808857fb50f53e1a71e5146032 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Fri, 5 Feb 2021 08:24:17 -0500 Subject: [PATCH 051/175] remove trailing whitespace --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 52f5476c..03805e28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ sudo: required dist: xenial language: python -python: 3.6.9 +python: 3.6.9 services: - docker - postgresql From 6a5752f025e31272fa735a5bcfa7a720eed1b9c0 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sat, 6 Feb 2021 11:21:21 -0500 Subject: [PATCH 052/175] Fix test_continuous subprocess count Make the prefix names just a little more clear. Should get one subprocess for each url prefix (aka "protocol://domain name/") in the capture log: ``` DEBUG idb.mediaing:fetcher.py:71 Starting subprocess for http://prefix.a/ DEBUG idb.mediaing:fetcher.py:71 Starting subprocess for http://prefix.b/ DEBUG idb.mediaing:fetcher.py:71 Starting subprocess for http://prefix.c/ ``` --- tests/idigbio_ingestion/mediaing/test_fetcher.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/idigbio_ingestion/mediaing/test_fetcher.py b/tests/idigbio_ingestion/mediaing/test_fetcher.py index f3525fd3..d2333be3 100644 --- a/tests/idigbio_ingestion/mediaing/test_fetcher.py +++ b/tests/idigbio_ingestion/mediaing/test_fetcher.py @@ -11,10 +11,10 @@ def test_continuous(mocker, caplog): def get_items(ignores=[], prefix=None): items = [ - fetcher.FetchItem('http://prefix.1/1', 'images', 'image/jpg'), - fetcher.FetchItem('http://prefix.2/2', 'images', 'image/jpg'), - fetcher.FetchItem('http://prefix.2/2', 'images', 'image/jpg'), - fetcher.FetchItem('http://prefix.2/2', 'images', 'image/jpg'), + fetcher.FetchItem('http://prefix.a/0', 'images', 'image/jpg'), + fetcher.FetchItem('http://prefix.b/0', 'images', 'image/jpg'), + fetcher.FetchItem('http://prefix.c/0', 'images', 'image/jpg'), + fetcher.FetchItem('http://prefix.c/1', 'images', 'image/jpg'), ] return [i for i in items if i.prefix not in ignores] mocker.patch.object(fetcher, 'get_items', side_effect=get_items) @@ -24,6 +24,7 @@ def get_items(ignores=[], prefix=None): assert fetcher.get_items.call_count == 2 msg = "Starting subprocess for" + # Should get one subprocess for each url prefix (aka "protocol://domain name/") above assert len([r for r in caplog.records if r.msg.startswith(msg)]) == 3 From 8d7fb550ac0c676905ca9bb07f866989b2f72333 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 22 Feb 2021 20:53:59 -0500 Subject: [PATCH 053/175] add notes for running ES locally for test suite --- tests/README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/README.md b/tests/README.md index 70c10ba9..3ad17836 100644 --- a/tests/README.md +++ b/tests/README.md @@ -118,6 +118,22 @@ A temporary instance of postgres running in docker will suffice: $ docker run --rm --name postgres_test_idigbio --network host -e POSTGRES_PASSWORD=test -e POSTGRES_USER=test -e POSTGRES_DB=test_idigbio -d postgres:9.5 ``` +### WIP: run Elasticsearch in local docker the same way we run postgres + +Depending on a live cluster for running tests is problematic for a number of reasons, including inconsistent behavior of test runs (see github issue https://github.com/iDigBio/idb-backend/issues/129). + +Consider running elasticsearch the same way we run postrgres... + +``` +$ docker pull docker.elastic.co/elasticsearch/elasticsearch:5.5.3 +$ docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:5.5.3 +``` + +To do: + +1. Have tests connect to localhost instead of the ES cluster that exists in CONFIG. +2. possibly pre-load a bunch of data / index. +3. possibly have that pre-loaded docker image available in docker-library ## Running All Tests From fe2cef4a8798bd2a1ea625b1d5f3e13adc419389 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sat, 27 Mar 2021 15:55:53 -0400 Subject: [PATCH 054/175] lists must be in the same order for assertEqual to be true Simple standalone test case to demonstrate: ``` >>> import unittest >>> x = unittest.TestCase() >>> x.assertEqual([1,2], [2,1]) Traceback (most recent call last): File "", line 1, in File "/usr/lib/python3.6/unittest/case.py", line 829, in assertEqual assertion_func(first, second, msg=msg) File "/usr/lib/python3.6/unittest/case.py", line 1028, in assertListEqual self.assertSequenceEqual(list1, list2, msg, seq_type=list) File "/usr/lib/python3.6/unittest/case.py", line 1010, in assertSequenceEqual self.fail(msg) File "/usr/lib/python3.6/unittest/case.py", line 670, in fail raise self.failureException(msg) AssertionError: Lists differ: [1, 2] != [2, 1] ``` --- tests/idb/test_helpers_conversions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/idb/test_helpers_conversions.py b/tests/idb/test_helpers_conversions.py index f26fdc0d..12995a22 100644 --- a/tests/idb/test_helpers_conversions.py +++ b/tests/idb/test_helpers_conversions.py @@ -22,7 +22,7 @@ def test_set_flags_returns_list_for_dict(self): self.assertEqual(['geopoint_similar_coord'], conversions.setFlags({'geopoint': [-38, 38]})) def test_set_flags_returns_list_for_dict_case_2(self): - self.assertEqual(['geopoint_similar_coord', 'geopoint_0_coord'], conversions.setFlags({'geopoint': [0, 0]})) + self.assertEqual(sorted(['geopoint_similar_coord', 'geopoint_0_coord']), sorted(conversions.setFlags({'geopoint': [0, 0]}))) def test_set_flags_returns_list_for_dict_case_3(self): self.assertEqual(['datecollected_bounds'], conversions.setFlags({'datecollected': datetime.date(1000,1,2)})) From 177563b2bb2092ed04c24e8f67c9b1198f5bd2ef Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 28 Mar 2021 11:28:23 -0400 Subject: [PATCH 055/175] python3 uses pickle not cPickle --- tests/idb/test_helpers_memoize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/idb/test_helpers_memoize.py b/tests/idb/test_helpers_memoize.py index 2bf98995..a3c25bbe 100644 --- a/tests/idb/test_helpers_memoize.py +++ b/tests/idb/test_helpers_memoize.py @@ -16,7 +16,7 @@ def foo(): assert foo() == 1 -@pytest.mark.parametrize('strat', ['error', 'cPickle', 'call']) +@pytest.mark.parametrize('strat', ['error', 'pickle', 'call']) def test_hashable_arg(strat): callcount = [0] @@ -73,10 +73,10 @@ def foo(d): assert foo([0]) == 0 -def test_unhashable_arg_cPickle(): +def test_unhashable_arg_pickle(): callcount = [0] - @memoized('cPickle') + @memoized('pickle') def foo(d): callcount[0] += 1 return d[0] From 834ef2a625deacb63fa35e5c64e13fc4cdbd3ac5 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 28 Mar 2021 13:07:46 -0400 Subject: [PATCH 056/175] werkzeug ProxyFix moved to different library The import for ProxyFix has changed from werkzeug.contrib.fixers to werkzeug.middleware.proxy_fix since version 0.15.0. See: https://werkzeug.palletsprojects.com/en/1.0.x/changes/#version-0-15-0 --- idb/data_api/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idb/data_api/__init__.py b/idb/data_api/__init__.py index a09e1296..f2795781 100644 --- a/idb/data_api/__init__.py +++ b/idb/data_api/__init__.py @@ -90,7 +90,7 @@ def run_server(info, host, port, reload, debugger, eager_loading, debug, wsgi): from requestlogger import WSGILogger, ApacheFormatter logger = idblogger.getChild('api') - from werkzeug.contrib.fixers import ProxyFix + from werkzeug.middleware.proxy_fix import ProxyFix logger.info("gevent server @ http://%s:%s/ ENV=%s", host, port, config.ENV) app = info.load_app() app = WSGILogger(app, [], ApacheFormatter()) From 75de07abee950d23501fb766e9bc02dffe1b112c Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 28 Mar 2021 13:16:57 -0400 Subject: [PATCH 057/175] remove unused variable (according to linter) --- idigbio_ingestion/lib/dwca.py | 1 - 1 file changed, 1 deletion(-) diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index a4926da5..e287136a 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -170,7 +170,6 @@ def __init__(self,filedict,fh,logname=None): rowtype = filedict["#rowType"] encoding = filedict.get("#encoding", "UTF-8") - linesplit = filedict["#linesTerminatedBy"].decode('string_escape') fieldsplit = filedict["#fieldsTerminatedBy"].decode('string_escape') fieldenc = filedict["#fieldsEnclosedBy"].decode('string_escape') ignoreheader = int(filedict.get("#ignoreHeaderLines","0")) From 09b2c85e4274fd415e4bde4dbbc8359bbe92b589 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 28 Mar 2021 13:17:48 -0400 Subject: [PATCH 058/175] remove commented out print debug lines and other bad comment lines. --- idigbio_ingestion/lib/dwca.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index e287136a..0317173d 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -52,7 +52,6 @@ def __init__(self,name="dwca.zip",skipeml=False,logname=None): meta_filename = self.path + "/" + archiveFile(self.archive,"meta.xml") try: schema_parser = etree.XMLParser(no_network=False) - # wut is going on. see https://redmine.idigbio.org/issues/3042 schema = etree.XMLSchema(etree.parse(DWC_SCHEMA_URL, parser=schema_parser)) parser = etree.XMLParser(schema=schema, no_network=False) @@ -63,8 +62,6 @@ def __init__(self,name="dwca.zip",skipeml=False,logname=None): self.logger.info("Schema validation failed against '%s', continuing unvalidated.", DWC_SCHEMA_URL) self.logger.debug(traceback.format_exc()) meta.seek(0) - # print meta.read() - # meta.seek(0) root = etree.parse(meta).getroot() except: self.logger.info("Failed to fetch schema '%s', continuing unvalidated.", DWC_SCHEMA_URL) @@ -195,7 +192,6 @@ def __init__(self,filedict,fh,logname=None): self.logger.error("Duplicate field index ignored {0}".format(str(fld))) if '#default' in fld: self.defaults[term] = fld['#default'] - # print self.defaults super(DwcaRecordFile,self).__init__( fh,encoding=encoding,delimiter=fieldsplit,fieldenc=fieldenc,header=fields,rowtype=rowtype, From 789417b572d38cbfdb9fd069ef638db7f78599d4 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 29 Mar 2021 17:52:21 -0400 Subject: [PATCH 059/175] failing to get the file from storage should be an ERROR --- idigbio_ingestion/db_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idigbio_ingestion/db_check.py b/idigbio_ingestion/db_check.py index a38f05ed..ce803b34 100644 --- a/idigbio_ingestion/db_check.py +++ b/idigbio_ingestion/db_check.py @@ -595,7 +595,7 @@ def main(rsid, ingest=False): try: name, mime = get_file(rsid) except: - rlogger.debug("Exception in get_file") + rlogger.error("Exception in get_file, failed to download dataset from storage") # construct a dummy metadata record with no filemd5 so we can later write a summary file. metadata = { "name": rsid, From 03d7fc4030b2f3a3b5f379c6fb5be1224f1d0b3c Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 29 Mar 2021 18:05:58 -0400 Subject: [PATCH 060/175] make error message a little more generic --- idigbio_ingestion/db_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idigbio_ingestion/db_check.py b/idigbio_ingestion/db_check.py index ce803b34..3d175726 100644 --- a/idigbio_ingestion/db_check.py +++ b/idigbio_ingestion/db_check.py @@ -595,7 +595,7 @@ def main(rsid, ingest=False): try: name, mime = get_file(rsid) except: - rlogger.error("Exception in get_file, failed to download dataset from storage") + rlogger.error("Exception in get_file, most likely caused by a storage or database connection issue") # construct a dummy metadata record with no filemd5 so we can later write a summary file. metadata = { "name": rsid, From 426f85b80516a5f8cf07cd7d40201bec19c5c9b7 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 24 Mar 2021 22:44:35 -0400 Subject: [PATCH 061/175] avoid the redirect (#136) --- idigbio_ingestion/lib/dwca.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index a4926da5..fb332fde 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -10,7 +10,8 @@ from .xmlDictTools import xml2d FNF_ERROR = "File {0} not present in archive." -DWC_SCHEMA_URL = "http://rs.tdwg.org/dwc/text/tdwg_dwc_text.xsd" +DWC_SCHEMA_URL = "https://dwc.tdwg.org/text/tdwg_dwc_text.xsd" + def archiveFile(archive,name): metaname = name From 0d96a9c0b6228479c774feb5070c5d2198e2e8c5 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 24 Mar 2021 22:48:36 -0400 Subject: [PATCH 062/175] remove old and temporary comments (#137) * avoid the redirect * remove old and temporary comments --- idigbio_ingestion/lib/dwca.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index fb332fde..6ea3eaa9 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -53,7 +53,7 @@ def __init__(self,name="dwca.zip",skipeml=False,logname=None): meta_filename = self.path + "/" + archiveFile(self.archive,"meta.xml") try: schema_parser = etree.XMLParser(no_network=False) - # wut is going on. see https://redmine.idigbio.org/issues/3042 + schema = etree.XMLSchema(etree.parse(DWC_SCHEMA_URL, parser=schema_parser)) parser = etree.XMLParser(schema=schema, no_network=False) @@ -64,8 +64,6 @@ def __init__(self,name="dwca.zip",skipeml=False,logname=None): self.logger.info("Schema validation failed against '%s', continuing unvalidated.", DWC_SCHEMA_URL) self.logger.debug(traceback.format_exc()) meta.seek(0) - # print meta.read() - # meta.seek(0) root = etree.parse(meta).getroot() except: self.logger.info("Failed to fetch schema '%s', continuing unvalidated.", DWC_SCHEMA_URL) @@ -197,7 +195,6 @@ def __init__(self,filedict,fh,logname=None): self.logger.error("Duplicate field index ignored {0}".format(str(fld))) if '#default' in fld: self.defaults[term] = fld['#default'] - # print self.defaults super(DwcaRecordFile,self).__init__( fh,encoding=encoding,delimiter=fieldsplit,fieldenc=fieldenc,header=fields,rowtype=rowtype, From f756511a4d67106eed07b9a30ada19b60753d89b Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Thu, 25 Mar 2021 08:50:18 -0400 Subject: [PATCH 063/175] download xml schema before trying to parse it (#138) etree.parse(URL) does not support https locations. It does require a file-like object. Download the schema using requests, turn the content into a file-like object, then try to parse it into a usable schema. --- idigbio_ingestion/lib/dwca.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index 6ea3eaa9..4de18df1 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -3,6 +3,8 @@ from collections import deque import traceback import shutil +import requests +from cStringIO import StringIO from idb.helpers.logging import idblogger, getLogger from .delimited import DelimitedFile @@ -53,10 +55,12 @@ def __init__(self,name="dwca.zip",skipeml=False,logname=None): meta_filename = self.path + "/" + archiveFile(self.archive,"meta.xml") try: schema_parser = etree.XMLParser(no_network=False) + r = requests.get(DWC_SCHEMA_URL) + r_file_like_object = StringIO(r.content) + parsed = etree.parse(r_file_like_object, schema_parser) + schema = etree.XMLSchema(parsed) - schema = etree.XMLSchema(etree.parse(DWC_SCHEMA_URL, parser=schema_parser)) parser = etree.XMLParser(schema=schema, no_network=False) - with open(meta_filename,'r') as meta: try: root = etree.parse(meta, parser=parser).getroot() From 84778039e830f4ddebcc5b312455ca843811bec9 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 28 Mar 2021 15:43:18 -0400 Subject: [PATCH 064/175] update derivatives blacklist with additional bad media (#141) --- idb/blacklists/derivatives.py | 2120 ++++++++++++++++++--------------- 1 file changed, 1166 insertions(+), 954 deletions(-) diff --git a/idb/blacklists/derivatives.py b/idb/blacklists/derivatives.py index 4aca9e42..09ed8954 100644 --- a/idb/blacklists/derivatives.py +++ b/idb/blacklists/derivatives.py @@ -9,958 +9,1170 @@ # Note that this is a Tuple so it can be used in pyscopg2 as an input to "NOT IN" DERIVATIVES_BLACKLIST = ( -'000281ba1d5f8d068e3dee96296ff102', -'0029b39c964e3c10ec92690f1835de8d', -'005481bf6fba9b3f6e43cd4dfb0fef5e', -'0092c69f4fb681c7a0859d4bebf9f212', -'00bdc846484def25260094da28c919b7', -'0135a964f0fc8ac95fdd51af2ee3d675', -'01885aec48c675b8c1cc5403538105e0', -'020cf7ee044d55a5a8b10b8f91caf389', -'0281374a2388ef6df6b45075ae6097e2', -'0377bc3723eabb98c2327f9646739a8c', -'039370bf32703882369bdc7e48f14a0f', -'03fe8c7fd5e80dbdb5fdacb28407f1a6', -'045687ebc08dc22d14b4464fbe935ed1', -'0516066e208b084ea10db953152a9b20', -'05239ad7cc38f0e8d3f05babf698b8bf', -'052bbad7b380d8683d85455db7221f81', -'05bc648bec7f4d0f90493cdecef2aa33', -'05e3041c0ee55331c355117c995000e4', -'060712adcfa6f7b040f701ca03a266a0', -'0628326e3e4e0fdd7699dd48831d4d05', -'069752bd3600e47e1f52ae8b3198d728', -'069cad0d46b60c05331f15dfd6e45b1f', -'070f9e2c5fc8a717cce67ee4c80472d2', -'073e4b8d5014976a94a7c86218e9413b', -'07627c8f8459308006b41d71090690eb', -'07c53ad26c54acdab5b6c98cf8e97122', -'07e4d25502eaf99ea34b0752cae1b225', -'08dc3a21e397c01fa375b93b2a87beae', -'092a6dd4f2986baeac98bd1849da4fc1', -'09369384ca82de49218cf5e67a14de6d', -'097f64d87922de3bf8f48b054ba844b9', -'09d1607af4f2693512d1bf4622a02fa0', -'0a16b7def7b7e0a5999236916fbfb7a3', -'0a2487d9639a1c56a9bec165e84f738c', -'0a4c0075b5d84c524054731716b54bcb', -'0a676e79756552293f8049fca16e8c7e', -'0a795cbbd79c99c3d1da81858fe91c6c', -'0a984427cb3fc0dcefae2bc181479dae', -'0aa080e93f6fbf0a7ac7a5b66515ae66', -'0aa6251fe58c0257846f9bc639243ac0', -'0adec962e4e2d84c8d09fa28cc6f608e', -'0aed12f5285e4b1621cff35e2e16f741', -'0b1d930aaa3ee98294757bcee48073b4', -'0b4bfb566529b7769fc41c6abbd6fa26', -'0b7a86a5e9d1d8da65fa98ac6040034d', -'0b8198767521bc2f452ef317e80545a5', -'0b9527cd7f1bb46a1d50595c4a92e8f1', -'0bdab840876ff65808e60cefb200420a', -'0c6cf45b5b98b535b7232ecb0043774c', -'0c8daa545af86a8d3694c51b45989981', -'0cbea54533054458364b597c3b888f47', -'0d26639585f7217e330c6355c95e58bb', -'0d52d047264a77bea36711c8bb65d20e', -'0de5c224f5790391238a995b92583784', -'0e2c99a8b88abcc2ef3d9f67d4f037cd', -'0e85728ab43a4e98eb038818d345b638', -'0e893b31a6866bd570b45efccc0a88a7', -'0ed8d05a51532399760374d7a926a683', -'0f5f3f1f6da28983f8888b7834861970', -'0f98ca8c7d66e01e1498fc62766c4906', -'0fecab2869153e06731c08ad93e7ca59', -'109a52779f4bb1ee917a9a4e73d93b5d', -'10a2cf81bab5ffc3b6fc79fc46b953b4', -'10b57bdafa2e54c48e5b52b8dff39b40', -'10ce085f9edbadbac9a5f316f8773a4b', -'114ab357e3f25d7607b8ffb29af26996', -'114f4ac6c63eacbe1ba2f9da392a5710', -'115926e5ecb579e8855980df1d57f17a', -'116e555755894ff0332128e74ebce784', -'117d58eabdc4f5eee8cb7a70a0004962', -'118f1f337541d5b1995d1534656b08a3', -'11ac4bbe9fce147954177b20f2edccbb', -'11ba53d81683d6cf9f44d245538ba488', -'11d49a25f1600139188bc0d19defa2b6', -'11f059f1872aae766a71995194e6b5af', -'12523482232547122f2f4a56953b3893', -'12b665c75d3e04cfd4c9fa548e02cf49', -'13891a137a4dfab120a35dd33159b7f0', -'13f781d434ffb6505324cace5cdca05b', -'14914453b3ce945683148bb603040bf1', -'1528b11b690e20bf60daf3dcce67e6fd', -'159924469bef2a400cdb8cbaa78df820', -'15bcd197946693d9893f1c5fa593c3f6', -'15e7ff8de63317c54471fa510c5ad56b', -'15f521580593d1a0c80332b563fe7478', -'160ffe5d0ebe9d7549c2293ac0c1d613', -'161993c546c061697950c2f562664319', -'1671e4d7647fe63663497f4492a95a4a', -'167b11e11daaa25e4009500b04a838fc', -'16c94bfa77976e244a02d291129ae3c5', -'17378c4626daaa51717369704ab36a38', -'179c38d57eb85b106dd95eba45b555fc', -'179da2793bfd142f33952884d20d21d2', -'17c73e7814a5f2d26fae015990d90159', -'17d4f2ff4a93e4644f0b9e9287f63fa7', -'18061935e6ab21f528d9dbbb36310a5f', -'180a7cf96aa37f56998a485cf22dc18e', -'1848993ceba448b05a1ff8e3a1ab427f', -'1856c914b879406212e82616cdaf8afa', -'18837494e4e25257e9d077d6374e336b', -'189a8950a57553c9d3bc75a4d8975eee', -'18e54707cb6f870fca10162d819386e6', -'196119691791be16ae0dc8cda65a2e23', -'19a1e7d12c3b6b5ad7de061d4d3050b8', -'1a37b248c0a76d5b9301b125f9294baf', -'1a678fa606dd463c7dba13cc7bf3decf', -'1a6d8da22a8baf42b7b1190b43fc16b6', -'1a85bc6c5f8780a344dc2a47df01118e', -'1ac67ebed39abe083407e98aa1c0eba1', -'1b82dc0bc393b5f8f5e4dd1ed050a194', -'1b8c2f498d2aaaf3136983ded141b5d6', -'1bb5175e4054393058019717217cf797', -'1bb909859399dcd2edf2c9ba49ba0975', -'1bca76eb57556639c84a6ff18f57d653', -'1ca233fd4979364b8488ff70abaccccb', -'1cb787a79926cce3b7cf9c4f66b34c04', -'1cd6bce4c44aef512873b29c629d1be4', -'1ce44bcdd93262a15336ad62d26c9b2f', -'1cff0213011d8f31d18859559e2b9491', -'1d165247b84f46e6772d1f659eb5a5dc', -'1d9f6266df45dfa7c38f27a1def193f0', -'1e725890a0bf079fb0624080eac60066', -'201a5adf58d04945b22229c716ec49cf', -'202d8ab0948b29b35fdac71b34ce47ab', -'202faffcdefdf3f55cc5f91d1604ade4', -'203feb124dd7abe0ff5115ba0fcd7522', -'209e5d06f999bac29ef6674531061ef8', -'21037e5f5ee9bb71d6445488a7322656', -'21124d7350412c4295a1a29ec5d85d2b', -'21788fae95ad3476eb4bdf4283d00ee7', -'21ccddaf6aaa908649541a933b71adf9', -'21e67b4405b8a0a9d487587a8e6392ae', -'22236ab4c15350f2efb39074e1b49dea', -'22c30e00ecaa6e99b925bacd41208a86', -'2346b30adc6129a2f8ceeedf47812228', -'23f3f8d03700d08eb60cd78c2a32760c', -'2433ec87a51596eb09e3e20d281817f5', -'245e25886fb6015c058f02c1dc0fcc6b', -'247e276ca082c52389c581d7ef3577d2', -'24af4c6467f9f1a2ae7a1599568838c9', -'24daa96e03038c673328310e4cb5d2ee', -'24e30c967af0e7f31175d45d52be2c71', -'250f4d3b1a2d0b8fe229d4ba1c74e99e', -'2524573dfc1342a7bfdc5e357924e673', -'254d91ecbc37546d38a28076aa20714b', -'255538ca6f6595ad001c04a717c4e546', -'256d9fd5f3e62d974ed31599f9ab0a7c', -'25adfa8c4f8f99b605d1fc77826ff4e9', -'25ff0de8b0465b991a4ca36accadd697', -'2669332932d27650b00c8a582c181650', -'26af6e3202e3b53d9c3c54927837146f', -'26b9d67b06ddc7a5e01603e68ba6b226', -'27a794fcd9ac4b04397597be791b144c', -'280a4af795570a047c785ef092df04bc', -'281d20d97facbfb9e049e9cae5221441', -'2864ea89bc97d03886948cd0c1017ce2', -'28b1de8ad599137214c2b06144f7074f', -'28bc6cccd51775ebbd86134547986834', -'28caa99622bbe4b37ed2950893636c01', -'28e55710154d7a253742e5b3b8050c27', -'28efed2d1f55020051fd19f6c0b4e378', -'28f735bcf1af085c4173c72b38196658', -'29aa9f4a9658875d0e2ffefd433e6224', -'29f96d669e31f3a31ba22c6ab3db18d5', -'2a252f1d230e71a99480bc8ccfffe1b3', -'2a270f3c2588e19c8c499e1ba403a213', -'2a289a4024f786d7c21bea36324edd59', -'2a51212944401e86c82e153f18de4563', -'2a575c5cb30e7999f7437b00402e4914', -'2a59bb4ad213b2bb421d1b9017beaab8', -'2adc6b2cf08b799d23673e6664932307', -'2af65154930f1e05e31f1e2af4a9c2fa', -'2b2249cf15c25abe780a55f9c6b18fd0', -'2b3ba80aa86e1066a408cdf63278eb44', -'2bb9d4e5247b0a6a2e51f0d4e9d59306', -'2c626e08adc15d0903f7ef7d0d425580', -'2c73f77f24194f7a08c458ff95da9f89', -'2c7ef33da8500df06ad85fba7a6c52ff', -'2cddeccf128d4268f01f244ebdcb2af6', -'2cdf46f102415b53307680bd8ab3c2e8', -'2d47023b43c8e2c5ef904581aa717231', -'2d65c10c6b1a40605dfb6654bee01151', -'2d85373c539558d7e279c40b00333b71', -'2db48c4b2a0291467b30d95f46a9f2e2', -'2ea0bbea18ebf4cee93ee8a8a8d23fe2', -'2f15aabe7689076793bbfeb80d79245a', -'2f30c1b9a7be022d22ce54d301065f4f', -'2fa0ab4a0f301da9807fa288b9b8b1b7', -'2fff098dc6663c0ddbdf921bbfdef6a9', -'30039517eaf9140564b07551395d4d13', -'303fefaa15e06dcabc8adb78d6c04dd7', -'30d3cdc9facf0c6548379caf5e61bb9f', -'30fe2bdb2780818e2a539f058cb6d0b5', -'313bf32bcffb5da88d86d0ed816901e3', -'3172cad333d9e85e7e972e08f1dd3a4e', -'31c142d791ead477361dab4678386dc4', -'31ff3a4ad528e7da897d5e4d61e4f5a6', -'323c8c13bacc55a9ffa52ae0754f4544', -'3261ed91de1201840519b4c8ba6e541c', -'32fc6fd1d664f514199b392e98f0637e', -'3342a225d55d3bfd1460d7d7b6235bf0', -'334d6cd24dc91fdf3c0e2dae814859da', -'3369461c36bbfc479582170fc1aba248', -'337987e09416de39c7fb1d1df21ecbed', -'33c7009b84786fa8dca3943ebe69f58f', -'3514c473a63d7c489f8c1406ebc5db49', -'35a793e9c9ac56d8c74352180d79812a', -'36140643574894602ba02382fe5b7217', -'36daab3778b884a95ef3c4ddd4fb7b7c', -'376f987ce35063dc19e641fe846d6bbb', -'37826389b91da6bc757f531b826d5b7d', -'37913e8e492234fb6b3727a66d77533a', -'37e1350b0bad6f9c00c5d2b897c64b02', -'384049db8d6fa59a9e1844c118a36063', -'3881f1922b2f6b0d351013213148caab', -'391a599b837ee4d3e32cda5ae8a01129', -'39287b10c6bfc48fe00be990f75e15c0', -'3981fe9769b1b77f871336221d1fbbb4', -'398e0fdf911ab0d48ec4fdf657be6a20', -'3a5ea124bed49fc56a0be60585380284', -'3a6ce9f7cd7b10addd593e5b44fcfd2d', -'3aa590b6cda38578111c926fce94ec96', -'3ab4b34d1a3c4b0a600fdd618efde908', -'3b251b703a9801ce242e87b0871bfbe5', -'3b3ae6b903843a01b027eacfc222bb02', -'3b3c1e5fd01561cf82ca898ab09918e5', -'3b7566ea6df8c193dd385f10496128ee', -'3b78089b7307818b2e8c2bf9e4a96803', -'3be309dc82576272d828183255819642', -'3c5a5946d3b86a85b64a46425a55714e', -'3c8669c0263e1542c24ad6604bd57786', -'3c9e1b23a00183a9b45fd8440b37aa4d', -'3d1bdad7dc8ad8ab4fa08defb46e62ea', -'3d1e9b80bafefd9ddd46260070a2cf6c', -'3d219f091f7b6a1ab87124adae2f7e94', -'3d877de80bb2dde6b140d9d82695e1e7', -'3e531f3bbf8bc42a18a19ea8937d33b6', -'3ebaff73c670b796d24c152d000d3ff4', -'3ecc7e5036a2933f3958298a9c632912', -'40e89f6f8e739454fffaad4148c96f6f', -'4109d6e3d10bd4e1b6fd44d9d3da6565', -'41362119e969765a9a0d4b1ffc1c9c9a', -'4157abf1fd5ff6f7cc55fcf03a289f21', -'4166ec33bb88dc0f023a654305b5cd55', -'41cb36d706901224acd7aeafe805b39b', -'426b213411b191ec7ea99a4f9a0a64bd', -'429c00232869ce5d2c0b5c53b3929ceb', -'42d12fc8429ed9b609ceaf22d7ac4a47', -'42fb1b4b447a64e9d1dd43168c7e1a29', -'434c79a75d496a440115bd55e4eaa2a9', -'4357de8bee11495b084b2bf16777e2dd', -'436648b6d4da9bffd87d31172ae58c9b', -'43ec92c8a31ff71cee5266cc06891d9c', -'440f084304e4bcfde47dc4e87bf84d2e', -'44318769765abfcd37e0d0c2aa5cd7e5', -'4464b81fc15b58dc0eda6031cbac520d', -'44b32061c4d34802d75b0fd8268192fb', -'45324979ffbf37e84ba58fd32b103d5e', -'45389cb5fdeb019f15f5ff6e4ce83856', -'455721a761d698b3bf6a6f65ddc5afdc', -'457741cead43310e419be4c798f028ab', -'457a39e18300a9416d48f93bcba36749', -'45c727cf7a54beca386c8efb6a849f6a', -'467e2453cca71a992d0698cb2b593e0e', -'4699e376449bcda91ee0aa62df5ee61f', -'46af92472808634424ae29d8038f261f', -'46be67f57b2a7c20e239cd65901066bd', -'46dbd6995d81facadf24fa7673bfdc5f', -'46ecc8679281cbbc2af4521dbefa3657', -'472736b6d1e2b0c2335583e3bf734eea', -'47377f181b8608343384a13bfb5f9789', -'4740e51bd8b298859d54455f215f8adf', -'47bc84399c3575e0270dbe8fa0236bf8', -'480bae5ffccbae4124285bf6276e0cf5', -'489866e361c55a874d0183470c0d72dd', -'48cfe5c81bce5ad75e2562c3e9b0c82e', -'4915b7a10e36ab791f2af54888b690d7', -'49500752f1de9219d6cd37a4fd189fc6', -'499fd9de88d0db2869593707cba0c9bf', -'49c39f4114a2ee21eda54a74a562bd40', -'49ce0c2a9cf277e01a2bcb36d129740e', -'49fb93a4190d73166a1287fb73db9718', -'4a27920d5831178486e182d4eca24355', -'4a29f53f1b11515179d5eeae4e975c27', -'4a2d95401099aa5c5ff4caa526da9e17', -'4a61fa295a22de8560757eeccea704b9', -'4b217a4d59cc41b06b33b25a9198a96c', -'4b7f671a97d283029ef0b60b340db4d7', -'4bbe3ca34195dbbdfe0dcdb2798bb0fe', -'4bcefd38b7e89f181e0ed93d2f88145c', -'4c8424ef265d3ac57053f79a179452ed', -'4cb6d0b4636b96453ac7243f43082593', -'4d33f14cbf2c48c8457e5b187e70549c', -'4d575bf111e79045b4b5e60a2ab73f25', -'4d917f40dc95c5162c4479349ba35fdf', -'4dbb515302b2edcbf29b150c00aa3e82', -'4dfc00bc70c68cce265eaf340c32acc5', -'4e8b0a809caadf218591824df58e5d16', -'4e8b4f143cd24dcbdb2f34d3f8cfb759', -'4ec07cbb6bd17196442e3c80ed769619', -'4f5a39d9cc9cce96cc89b21af5cf275a', -'50a0f4baae5ed69e7893b58f0fb93d0a', -'50afa16b0bdd5903e84499011755b5ee', -'50d0eda39a11627a7566fd931b1d7da2', -'50e2148bb145a6a22d7add40577fc655', -'50f53d47e80d9ee084dbec179bbff42d', -'51ed5dd964abdb8667a3f4a2164e58e4', -'5205026f9155d0375f0fae4b64aa5c96', -'523b1a3d1f42a90fada3df3c44e91d2c', -'5248afcb1be3eb36261f3d2bd1cccb0d', -'526d38a1768970d243c7892ff3ccb6e0', -'52b22af81f73000abf1812921faccfa4', -'52ef659713b9351c5f00b4260ffdee72', -'52f007b687a54d9317e97b55c69e05e3', -'530c067d4e8d06cf4c6abb9d54284ce2', -'5340931fa00694272ea1ed65a0b0142f', -'5357294e9f1e8137c7cbcbe93487e4c1', -'5383a0cc4f04cac5eae39bc97974fd5a', -'5399da7661e88940639467979ae1ca84', -'54113debe6196b329b52dd414c32ba8d', -'5485d6ce064e6f3ff9b77654faf3c471', -'54b166ba042e89c7484f675a370ab367', -'54b88637e43cabd1618a60fc15938ea0', -'54c8047a679d43200ef4ef39a99b0d03', -'55ad8d0bf7e02284acc6b928a918e225', -'55bcfd0805ea911d5b73a5f7e1b04640', -'55e1266f85b6139c1a3b63878a6352f9', -'562345a37c75abbfdb970fd78c700078', -'563b6b092c37a119a5a434afff399cf1', -'565de2adf4e331dea123cbbacc680d39', -'567155590c1768d6057f985ace6eab1c', -'56e5f7f5206eeadafbe9ff8a9e960416', -'56f3af901191f6edaf9de972f82ae99b', -'56f6bd72ea4de9214ac652bc434cbe6a', -'579641706b1d3df4fee1293648bd63d9', -'57b29beba489ef4debdf9dafd5c0bdc7', -'57d66d5ccc423f826fe628a57ce96397', -'580054aa2370647cfe9826ccadc4b96c', -'582c2ee88cd211bc9477a5523b27ac1b', -'58b217dc2545dccfe31b75f1b59e6e4b', -'58f4391217544ee4b92a186a07a41795', -'5911f7eae4da6bd53a082f9ddb30d131', -'591c101d6833c440210609b18095eb2b', -'5938266faebd33436de01ff1a3ec7436', -'59472321d270a4550c9ed4bece9ffd89', -'5961c455706158ac40a34e63974c90b8', -'59bf6438bcdab6867535e31803655a17', -'5b68f1fab59ce019284060e2ee9af815', -'5bbe967a0f721d6daa709316829ec97e', -'5be951da1e739ea740e53712d562d272', -'5bebad15261f25307a93dab063589a9e', -'5c123d7c181c4e3af1add44096af70e6', -'5ca3444c645de8ef87c4cd65d5148633', -'5ce9c8a3d34288e30d5124ce99a08eac', -'5d0f14c76eb1b2bcdbc1245e77c0b5a9', -'5d3d2c58ce79cc181e8dd46b39ad6992', -'5db4fb6d26d4277470ddb73dfc57ccea', -'5de1d6a94d55add2a177b345deac8df0', -'5e0a310921e2ce4065447a36dd581b91', -'5ea382d38405c2f28c85be72b9e4cfb3', -'5f198e1ef7dcc9a3eaaba6b2ba2af37e', -'5f1ca699f84fe2ff99c81fbd3f52ce1d', -'5f919c5345bfee0b3c759d7626ff9371', -'5ff6690c66a27f0b3fd2d4f85fd4fbd9', -'60253b7dcaac00bde953c510c0902691', -'602cb41449b1f7dad75677f83eb1078b', -'6062d276f90d81b84883bf42193c09b6', -'60a379d8fe9b2daf9b76d73edfcc54e9', -'613270e0f1f630b0034939bfa1058000', -'6178dd680e125f4e7b25eb12dde72f4b', -'61804ccb95db616851ce33a026b8a2cb', -'61c63de6086b72c2b13ed955220ae62c', -'61c722a79c087593321e43d08401da4f', -'61cd317e99c05839abc3b53a06295e10', -'61fd4070d92b8d311dd8771732df776b', -'624df4ede61e915aa3eea6d68aa0dfb9', -'6297f545b24306330c7c0db882de6578', -'62b132077d93e516948dadd6df437960', -'62b575dfeb7c9499b93e746fa5ef6d46', -'63cad32b83072d6fd80a176c04aa28fc', -'63d6ab6cd019cdfb2f895368662a906e', -'63e62d70beec7cee6d6591d72d7405fc', -'642159ac6f0e25bc36b445d1daf7cad9', -'646f822bb5d1b11fe82af1060cda3718', -'6477363ede1c6c7bbe8ff2728fbabdb6', -'64b4667258e9eeece9c12965e0509905', -'64bf323c9f81116146d7552422f35527', -'64f5c6f882e74c27712976ef9bb592c1', -'64f6bc5e6cec9bf8c4fb694c3edba5a6', -'6501f42529aa8db0473b7b0866df53b2', -'6649382d7c2d5d1b834868c5b159de25', -'66897ec932ec14c43b7156b453d9aad4', -'66adc632cd068063ff250b210c529f57', -'66d650bb662475c6b5cf052a3826ee9e', -'66e1ac5e2e8cf3cf445eb5da5f631ad8', -'670649bd1ed52cacf4bcca8c5a90fc0a', -'673d86d168833c4d1440c9248b877b83', -'6790ab2d34cfc19461a6bf255f925597', -'679647e69a20bb6482abd1e5f2b7d931', -'67cdacac202a2d3b16c1b22733996de2', -'69143bfe7dcb9c15fca1160c3730bd52', -'6929a033e96fde056b29ef49423f9f57', -'694bee21f880e1af4bc78315cf7c84ee', -'69602a09232ca64ebc83156a4ab70b5a', -'69c0fd411df6a5340681e4157f89864d', -'6a1d9cc5886030d429ec810f31fcf65d', -'6a545e8466d45e68fc83abcee91cf843', -'6a6ff872d323e4a864cd4db863883e74', -'6a7ec4e6c8321662dba76276973827ef', -'6bc73fa1caae19ba8bf89cd5d2aa0caf', -'6bd91c969b4cbf750d6465852617b33f', -'6c1124e6418cfcc8ec970f7d38cfb280', -'6c524f8f4db28ee0c0f8d46e94f17243', -'6c8825d743b0cfd9de0c0e03d594f667', -'6cd0a5be761684710a177d11293df197', -'6d788689f2916b02dac731ca91138634', -'6dcd61aac825cf079eb911b5645c8b4b', -'6e5688e04caace69083656ff88c5df98', -'6ed89547742e0267bf49d6f94c6c1c3f', -'6ef8f1dc83ebb978e32cabb51ec66dd5', -'6f01bd45ef491ec5492dfb0e476bdd82', -'6f2d0214628cc60a977b21a2621caad9', -'6f3bfa56066daa8a6c7d9d252f8022f9', -'7005a18a68c3c4e2ef558e0e25abc47b', -'706b4c2c7609f707a135a31503c40636', -'7088da1b6be876dc5ae19afe1d37833c', -'71c6b51b202994bc21f2ff1fa543bda8', -'727e7769ca161adda9fe76fcb4539696', -'737e7418319e6a03ec7af0e6ef567024', -'738d64465529ce383debf38d7f9d63ce', -'73b7f4b873e41a7d7f1aca5101e9ef3d', -'73dcc00247460cc29323bb0403fd2193', -'744a49bbc35874854c38f44874e25ab6', -'74e92c0b5874a32d5346c9fcee48d54c', -'756113b64beaca5a229f63351275d7da', -'757c44ab06beaafea129220bc923805c', -'758d6cc8e426eb18d8359259a745c443', -'75928e20bc8a672b078e9ed15cbdb012', -'75c6da2c629e93a98bf1c421320ac78e', -'75f352bb8d09891980b48eaa9c555df6', -'7643a460da7a5cc72dff8683f9261fe5', -'76c9538fb44b182305cfe06bce246aec', -'76fec598f95104a6b56dba11ef99caf6', -'770776a4692cf4ac97f54bb87db53eea', -'77715ac7a4c5c96c655489de48341b67', -'7785aaca0d2e9b297eb71bde93cac0c7', -'783a892e7a29f9aa23100338ef646b56', -'78bcbe0556dfcc0cd7089c18d4289387', -'78c924a5e451c517f3b2d4f1de1ca114', -'78e67353cd5fa9252b5984c7fae1632e', -'793aea3587ad8d9117d5ad3d047eea58', -'795f10876dd8e00f7625c2c2dbdcd98d', -'797cfe19dda9b2be011f7d8520dfcac7', -'79886d8ba404d4db4afbb4bba89e59ba', -'7997118bfc8baf77d9e145b331ef1296', -'7a7ceaf45da6ec49d0a3da6d1c2a8bd6', -'7a995acf71149384c6e0e7b94efcc8ea', -'7aa009a271f3f8f12b85424edb75194d', -'7b4fb468d023a008e297734c80e26845', -'7bb690b68c87d2663e4eb4fef1956b5e', -'7c6a85e8a7d400a31cf4610e624f0042', -'7cf0e55aa3f54e3185ccfe5553cce885', -'7cf7bf9a344208ddead805bca022455e', -'7d38cd67ee31bba902ed0b6a48cb871c', -'7d998d6ac8b78e36c9307dba631c0960', -'7db2c942f5416731432ab46c502fbacf', -'7df77a8d46e5a1de2c28440561f26b0b', -'7e1ca1be21917fba17cf0afa8286ad57', -'7e7ee6ff06a20f3c2486be9d2908eea3', -'7e83af8f970d32511937ff32196bf4a2', -'7ee8d4be62fed20982ce793a4a064d43', -'7f0a6dcfee36dcab6dca4c30b20adeba', -'7f3aacc835e7377eb31d8a719646a62e', -'7f6d0395a0616650e782b01404f814f5', -'802dd07b381ced45ea3ea7fff94c6520', -'809012d6e0e43d75de23d4dd518818cd', -'80bb14dde7d082b3d3fb9478a4d0a1d1', -'815688a1e21e9ee9ad70eb5f3ae5a9d5', -'81d1cce893f8c09876df98a7e6c073fa', -'820f2227d09fef6916f91af9612eaedb', -'823cb21fb8fda2a217311572446a8159', -'8242ed59320fd52d2bebc8c284defac8', -'82490711b8f979a9baa35569abe31a88', -'826a669dd3e643deffd12cdc07b6e5b6', -'827409e108a5998219799f6d75ae3ef5', -'82e27ca093879d9c52178eca9873ed30', -'82e4842c014e236e790e2d4d98e178db', -'82f2fe481b281646944503a46775e3e6', -'83c7aaf82084d700a68da0d94f8ae1e2', -'840faf7fd3be8156c8ee4afacb9c43f8', -'8418744e36de859dfeab8baac8497170', -'84205ef9e245ea6d115ff15a50706a8d', -'844c7b90e45d94b1233901776792e1df', -'84591f5c57c3bdcf86e6deb00957d906', -'8489ead9e6e14764a936d97e97d8679c', -'8514386f4a79eec3307c3c0dbd943572', -'856fb427e0cfb74e69ce81a51f2c27bb', -'85a4bf0026a29e265cf6a8c93c904dde', -'85bd68465045078e8c8998f7b3e05d4d', -'85f86fb05282cfb13c779639eee79041', -'861687d0cb74dc7fc6bec0758253e172', -'86393e2fc8cca145ec225c1bd4784e2b', -'867d0bf6027f23a1a7b6ee7f22cb0334', -'86ab964396ab1b1369474f820e439aaa', -'86f2b6fa9fa6e5ce9daa568984f3bd39', -'8708eab5ef80b6aba0701784825fdd32', -'8715421a9f02732abc62a296442b0030', -'872795e5abf5b3adc08c33d4bd2450d7', -'87e1b58ae4a4959848283a0247866407', -'888139cef0fb89bf3f49258e5ef68f4c', -'88c7f35d0e01ec843a5fe6d82568408d', -'88f3b2721bff952eb16f0ebdab8aff34', -'89041e1d6532461ea5290ecf7e63c9e9', -'89488afcd36ac32c229dad04fbfc60f9', -'89e6ab19dfbf51aef39a36878256d937', -'8a1d2c6f04d33de4de0ff0808ef93287', -'8a4128e6e64f12ec6ef2a3e0497d6113', -'8a4a460f6d2a013dce404a896f822ef9', -'8ac77de7006b2d18146ab498477fe631', -'8ade6b421f39a52258845487cbd02485', -'8af1bdd3d07f165b0a71ce5b33a89cca', -'8b265646a5b1cb8a7313d7cbbcba3cc6', -'8b489d03c50178fe9fdf3b70a8c28761', -'8c20be197ec6551516e8c886b0cad945', -'8c8bf61bcb408e7ab96b5deaacf4ff97', -'8cad6ddc155bf1f52a75e6b441de0874', -'8cb7507ac9249ac82ee703346fac7de2', -'8ce6bd7e475f0272889e25e8660ebb66', -'8d119f3c73318c23fb9a13c74f9278a3', -'8d26165011a96624847534d6ff1e5d35', -'8d4d0ca7699a9f02311aefc59abf8560', -'8d650252168d646e8950d9f94d25c144', -'8db9640ab917733da91c1c2b7f023ad1', -'8e0d470aff1dd8ddf1d52f168ab7bae9', -'8e1304d107dc9ee1177470b1212b312e', -'8e1b12aaf505104d6174732d8bfa5fef', -'8e5cfc9a8dbe1aa81d74f6876e2bfb49', -'8fb7358018491128d5ab06a33456a337', -'90d45b6d3c7ff6ae9c0b6fc9148873e8', -'910441aef51561bd628cac1bda9c5121', -'9136b1a098edc349a159487b6034d3dd', -'91def347ab1bcf83762acde09864d44b', -'923583c00c9d0bee9c9b110784d4810a', -'925a9449dc67d6e0e8f5b3595f72d1de', -'92a65c0e0fd9a425a68b9e8a20fcc182', -'92d9d67ebb769dcece2a8f31339b4e72', -'930944ba677b49e6f4e2bb7774744703', -'930f6fc2222a17b6611a2b2be2a83353', -'93141fcd73b55e0774ef4265200227fa', -'93195cc5c1306454f94f7e506a302826', -'93602daa68e8715c30400b6db88116c9', -'9391e2d9febbf06d0fac4df187373971', -'9395b7f1bd3c7b642ddea6abd8b62f82', -'93a91754c07693d02c6eb9879e529137', -'93f02123e34d4bc0af11365b621dfa8a', -'941d10057afc815b397ff0669a399891', -'94ab856bfde35cc05e1b9761274ca6b2', -'955445b85f03068151dfd835d6674408', -'9580fadc098f33fd73769a81c30dd6bc', -'960052f338d05917d374fdf010da27a7', -'965f0f52e11622d497f25deb3d2c9ceb', -'969310ebb630b2bd582e66a40778f31e', -'96a79c335d0141fba7f36f2c4d40452e', -'972a908e51bc810920b6e0f0faa03f39', -'97419977b29bcf7072fee7023a940834', -'98541f1ab176eff543c32aee7e3521d7', -'98a5765ef891428900492b189458f55a', -'98b3f2bbc5bec9590b126b61a657585a', -'99025f3b4978e2d75c6fba6f0bd295b2', -'990b13c991d336897b0a35ba96c11fdf', -'99674e03dd1b404963437cc4db8200cd', -'999538fbee76a7397a4dee789d5217c0', -'99c5b2749fa7290632dd67b751a43474', -'9a791f306d3c493fa28b360d036a6965', -'9adc0a0e90fc7428bd7d78a081543d90', -'9b0153790035845f127637309a72fdfc', -'9b0f1f8d0c17aee0b49df0382958e471', -'9b2abb24e4910f3413f7fa1a098f06c6', -'9ba5ab5bf0ddbc6db4aaeeb0a9a132a9', -'9bd0c13645c8b12fa8823b42877d001b', -'9c02a9e7854d7e8076dd2fa7af65e9a0', -'9c0de3d7bd3b230a2f2d749aa8fe7632', -'9c613c75a7d9e3b369c3a8a90e6fbbee', -'9c7ca3b5efe8cb272e695cb4ba4e5df9', -'9d36b192f819354ccaac8156e2aad709', -'9d8bd93d97213864176583353b5acaf1', -'9d9c0b6a43bd5e2e847465a7e369c2ae', -'9dcd0cc82e776eb35be598efdcdfdc9f', -'9dd550e9af88a5e7592caff6474a565e', -'9e02e924712fbcc9ab8b5fbad609478e', -'9e085ff38b712db252458208881a1729', -'9e5f54988d48f5adeb352cf4944d4e61', -'9ead6071933e9bd5390902c1a9ace0e5', -'9ed796b348127ffb1718f9d8a8de314c', -'9ef248745ac080ea599efe1a2fb46b4c', -'9f00306c6f7766ba105e8d220964017f', -'9f3fb30f861efc69a5960e65c7e1ea64', -'9f7e41c4665e1264e52c6edfdf396d8b', -'9fa88094b923244404b1506ad408b095', -'9fc7b291a1f6e0df908682f459651683', -'9fe2c8a78837817f221c44605f3a67be', -'9fff53b2cf73c6e958ac10207ace1473', -'9fff5845bb6f25a59dbaf6eb46ffe216', -'a0058191941088414300116894889919', -'a03ad1aada4ddd805c115b4cdd4d05b3', -'a0a85041965bf816e67114ab38d29a6e', -'a0abed53ec7a3fedb163438e8be1c8e8', -'a0ec5c139564054719269fb275c7b2a4', -'a14dc50a9445da5bcd76ec37180d3c2a', -'a176053c9f21ca729de393429ed835dd', -'a19c57a191b182e614a1ab3002d46a00', -'a1aec2fd6ccb57ff4d264838bf2702e6', -'a214aa6ee2e6c3ca82516f4115811a5e', -'a2bcb8b69acb92a3fdb7074c4db149ba', -'a2bf9a0777517984dd0ad65cdfe5d4e6', -'a2bfadadda18c52392b77fef8ae5824b', -'a305fccfe55b81dfa19774a704fefe98', -'a3a33f3ab2183f9b1ba1dc30ae2af2be', -'a424d9f189c3e47fb7a1b179057888c2', -'a435ed22caed17ed51c36f7e0e7a9610', -'a44f8dfa78f7151c61157ee41010b096', -'a4572544706239f74b177280ee98b0e8', -'a4f218de8717713d2c8c7af1f1d43325', -'a54bd6dc1f576c4b281e94a9538312a8', -'a54dc2359a93e830e6d0613f15741b33', -'a563ab6f6293403dd5e2f9dca8e7fca3', -'a59a6e2b09ef72445deec90902ebac84', -'a6d406552b5e29b93518170516595255', -'a73ade7e0c839bf62cf01a9ac07071a3', -'a74e8925ae11ec4dc13796d995e3c761', -'a761ea482c39c292ef0cf0f78f411882', -'a7c6e8594c4052b2364f646a13f16239', -'a8d6dfc9d28961ea1bb0ac5d4269f89f', -'a925a5bfecda69217ba9e8b87c22cbc5', -'a977b3ff47770e00ad509564e6d2af31', -'a9b5f0fa1f8218bd9a75ae2d0c316a8c', -'aa4d266bc1d34b8a266a23b38326802f', -'aa5057ebbe002f6f4b1c6df187d40903', -'aa6691f2eab7d28f000ef94cf83175c8', -'aa6e372b4ab9bc8d8308d753267a0664', -'aa950028a9ba45a06b024dc6adf40d3b', -'aadff1032abd52b873442ddec72ffb9b', -'ab637d25fe43d54e4167cb48e7b7dbe0', -'ab98554ae0f660d968427bb08e634c14', -'ab98c85d25084701a8b0ebcacbefdde9', -'abc0cbf11335d8679cb31b0edeaad57e', -'abd2013f87814e3d939b7f4f6a959750', -'abe3be940fb2e6d1150bdc8027184b79', -'ac12ef0c9039ba99201430c56fca5760', -'ac2a8597bf26093cdff8a95d38ec1657', -'aca1a6e5ee4f52d7651d664b88dab486', -'ad368531805ff9dcb8afa357a52506f5', -'adfbbe138ebfdfcd11627dd715c6457c', -'ae0da1761638805a9d3b2c4efa21c411', -'ae1218d977ad1e8f865aaa4bd0588836', -'ae3516de7256743ad9271659b00b52c2', -'ae93620e68fcbb9e746c6d782a58f320', -'af97ad66d2b2b12563839014e201f5d6', -'afad69f8da5d9e40225536d15bc54799', -'b0784a720da6178453b0e09c1d7cad08', -'b0afc9e3b56ba7625c74517a9ef119db', -'b0f69fd6a03c5ed7f2ba3aecfdbfd36d', -'b14f07fa654ecf4445bb47e76280cf5c', -'b19e199b63ec110471db997701f5f69f', -'b1ae7dd1af59b1a5a35ac610cfe38975', -'b22bc3afac5b3e42ada8c2a107c3a7b3', -'b28f9d7236b2b86cb185e78cb11d75d5', -'b3317fff300745ebe79cce6ad4f9ffdd', -'b33a769072368af1e5517a396cdb6f72', -'b389513ef9b573f4fb66621d88be03f3', -'b38cc02786e5ad09c0376d27a92bf696', -'b3a5a3dfd425ab4ab2de05e606741ee4', -'b3b4b56cb536a7e30002efd478ab3470', -'b3bab6eea5975f779bf77c39e594e8f8', -'b4363f44a7566e28b6ef90a436b82ce9', -'b4af2297a8c78cf7e4be76b4332a7465', -'b4c5a1d67861bee42c906c23b8b4acfd', -'b5d201196a047162e0794600f7e44e72', -'b5e9797476bf0a8a88799bec048ab6f6', -'b5f64a432406a049c70c9c351a584672', -'b6f30b812fb156fbbe6e3e38fe9f8d3a', -'b70397bed451741a08d16453ff9e4eab', -'b704af0cac0d3493a953e4663c92bcc4', -'b74d6c88441275281fe24b11e5c6fc2c', -'b7d9ac45b117cdb3764c1a469cef560f', -'b7f9a6bc8c7c97b0f371c757b806d60e', -'b85ebb10b0f74d4c9714cab8ca9c3fbd', -'b873f4c82759e30af8caddec96fec759', -'b88ebff952bc505b1aebbe11035a509c', -'b8c7e6fd8c05ff2a59e60d335e4c48b5', -'b8ea27d8822ceb979ebad1bdb6f75fdc', -'b940b555fb25743fbd9c2da67002f347', -'b9e3f973b4d308e8678eb0424154ad22', -'ba59351efe64eb952c5504eff3627367', -'ba5bc49592c5bcde492cee85a2bd2ff4', -'bab707b152fbf16922d8a97626a788c9', -'bb2d9b8d7080ce4a03a91434d123dda9', -'bb4a52d05801d7bf5892432f20488c3f', -'bb72ee39c524fee8453c89f326319dc5', -'bc35b3b8bbe0860e626314849a13939f', -'bd02018461e4650522af963585610890', -'bd1de32462eb83067201e92f07c355f7', -'bd436cf8841b821baaa0bd3368154c27', -'bd4a764d6deddbb1f63946cd78b8b629', -'bd57025f1b5e4f67c547944ee28f283e', -'bd6edf62fb0ece06740d68da4bdd847f', -'bd8163865f0e2624385bcea221117654', -'bdd4053d9f756a88f793e5970ae1c4c9', -'be2e73cb632f5abc02747ecf37b664ca', -'be33c1ba3055c5f8cdbe7d2f1794aaeb', -'be56d693d135895f90e8d591f907238e', -'beb3449e2b27df9eda77bf72f658fe6d', -'becdb68b1fbd7ef0435c2cbdcd916a3d', -'bee71f4ce1a551fadbc14b2839f3add0', -'bf462d40ba86abf0b878fc2ee4e9572e', -'bf68d41b1e246b244d42d8e66a1afdae', -'bf873fa981d4c5205bdbf1381a601f65', -'bf8bb7ee945c95eb699d1e8ebd84913d', -'bf8c113c1d4ffe3e494b54b527d8cee4', -'bfc027914b444b0e2064b97ab90e0790', -'bfcbbad7a1aad3479b9c703f7e047d37', -'bff9378e4847a6b18473f158c5517278', -'c0307c0f56136e43e3c46ca94802ea3b', -'c0477075243562f57002d116366620e2', -'c04a24e5d11b2d9739608c36aa6abd6c', -'c05360d50fd2160cb50fc30af4071d6d', -'c0935c7edb865554aacfe7caf65439ab', -'c0e34351819368943bbb1806e3e8b05b', -'c29e8e5b1abfd35ad853d658a051f526', -'c2b98ce479e0067f6b195324065e3153', -'c2ba678a8326253e76c93bbc10134e32', -'c3820aed58d03d9b6cba57271022c3b7', -'c384d18a53d18e1a15c6a56187bf99af', -'c396853fc72d158d8676e6ab02ce89f3', -'c3c91eab72d942641f8850db5b9599df', -'c47873b0d4c8c052c4f370e4d38cc3d4', -'c4a454dc73ddbacfa3204813f9d7dad3', -'c4c95187f4a10e9b6051dff9df6b42b8', -'c4dc5fd4bba266fae6d4f75d2654d112', -'c4ea1cb327db64ecc7635ee472f2cf68', -'c5fed80fcd653b373153fb5dc0ac48b8', -'c65c4daec0dd7ac4a71df26648ba1671', -'c68cfb94ba8b8d21be981ce34fd11dc6', -'c80707a675dcfe089bdbf5de048e2f1b', -'c80d70c0335e77ae67689831169bfe27', -'c8b9e9141ef06b1929c752caf5caae50', -'c8ba4601368e3336e7bdb86439f2eda8', -'c8e44d119508941687b6645736793123', -'c8fbadba56b8674ef1dcd7628ee4aa0e', -'c923f1176aedaba9accb908e479e7423', -'ca070d23796d6ca6423164c8897f0ec1', -'ca0a1499e54d430ac19620ff4a1f9ba8', -'ca34118127cbd75716923afda0222efa', -'ca7ce47815d2f3f1599fda16bcf5043b', -'ca88872a712dc75393b70555f014f70c', -'ca9f16470374ac7da820a6fe1069239a', -'cba0fafd976a15454819e12933ecf936', -'cbdcff00c61583142b8c201b44d48273', -'cc5246a073f0a08d56155f053910f12c', -'cca7cbf3311543395d1b96780ea0e982', -'ccc4aded16cf1e5fc3c57863ce6a70dd', -'cd002cc133019c939468d7f524573f43', -'cd142f615ca8c4bea7373e7de06168d5', -'cd6fb6b993c252e2e63ef2cd501072de', -'cd9bfc6b7fe5606f4176e921f4770224', -'cdc29ef82126350af5d336e83bdd48df', -'ce4ea3fcebd1f30367445122d3bd31b6', -'ce95ecbbc8b78cbef52ab86ca4888244', -'cef2fd47dde23bfa38477243250f1c5c', -'cf7a0f2620142e307e032e3daab72ff7', -'d06423a5a0e7c00a97f71c5645836ba6', -'d0bd37efeb735c0ca77ceb645235beb4', -'d0e7e1314baeda954070777cf20cc724', -'d11a8c564fdd0c65ad98f9aedbc51661', -'d12db17728205d9ba6d9c87bbebe58d3', -'d1666b0d963c473ee075c73598be7053', -'d1a7df95806255ee47d9ad96aa0e861b', -'d1b31eb2c1a3c7c5750ac5915f1d9718', -'d1bfeb98ffc67d02311455c5a3cb49b0', -'d2189246794299fc353137bac7a59f2c', -'d23a9506c8af058d97e344bbedb341be', -'d24198307be9be78e5de33d8793d41bc', -'d3caeee2db9be23950fa51844e75c7ca', -'d40b265e06549e76a8f637422cf0835c', -'d45268719a8aa88ae9f28fc27f07c54d', -'d49e5340d7501f22a6fe46bed009b7be', -'d5334a6b570226cd184f5bea1a985d80', -'d53538982298230b04403a695dcb2231', -'d583478857fc9e12b3d538ca3b65d5d5', -'d61233796167efb681401cdd908f0f70', -'d627b5f50bdc0fa915d39cc32e7b760d', -'d628ffd8aee1d8dadaf1864d295f3445', -'d68eab15bce2dc4af3a92fa815f59336', -'d6ada30b4cb916b077d618a73a93cf04', -'d6cc7d0e0120a6a267adac3a05f6f287', -'d6d08f04d065382933283a788f52f080', -'d70da1d4fd5fe85e82e660c183da108e', -'d7c4dc223f2a4b9817b7d30940bec64c', -'d7de5eb5311c6220975dedf108e46a39', -'d85b9548571a9d0edc73590660c0c18c', -'d8af9ab641b190aeec73d211dfcbf4db', -'d8b1e3c8d7a051c927c90ae2692b3768', -'d8d8bfc0724c93fe35791942f69dc64e', -'d90bdd41130bfd946f0309360a7ea05d', -'d9109defa43b38c20ad21da38cf2c6c9', -'d9199654e9bc0c84ccc64c81e0d3c393', -'d960f8a4fb9d37d457e5a4b46faf98d6', -'d9b35f8ac019ae98454b6e98c279ab13', -'da108765256f11e4edd9314e1e494fcd', -'da3719df7e7e02d3b78dfd10531261d4', -'dac7a910ae791d1298a7941662ef0eeb', -'db28722a0d1a5b0c5b1e321dfd127195', -'dc256ab79aab41f9a65bb72b9a53a154', -'dc3b8b030492260496a8e05b7637fd4b', -'dc90b8aba25f31d80df5c3cea85685d6', -'dcc15b327be19bc5b50e434690d35350', -'dd92bdae4ed7cb64e16f782d4d7b7767', -'ddf1d2901866aad80dfb033e6c60e601', -'de1dafe1fb480a101d7da69608e77074', -'de6c1c7dc6cba47e1dd3b0ad207b074b', -'dec43b4d74667467c771fe7322dba942', -'dec8b28eb5eec90f9505e0ed7064d294', -'def4ce670fa5b300eb7b677a5f193f28', -'df10c3db0b4a41946fe5729dee65393d', -'df3fb9fec424fb443568c2ccdc0a7a11', -'df659e4fce1b8f5dfba57d775013bc52', -'df8468e5ceefe03d74e3c4fc52a93a8b', -'dfa35a09f7e1f58e5b766576ba3cba6c', -'dfade29220faca203263212735beb7ce', -'dff26f8c18ef6963960f929c7426e469', -'e022129a9694b8e72c8870552b7fb277', -'e06500ea7c2b0a626dededdf40ffd65c', -'e0852d277488e972e8785a65f2d13fec', -'e10ebeb92e7a3b51f97ae1cee8315622', -'e11428b8590c781a531a3845c81c92b7', -'e13c3af20606152ffb2cde63161debff', -'e17a3a67838ce6bcb3d932ee6ac70965', -'e25bbb37db9243c70cbc562e357bd2c4', -'e288e6095032e0639f9ed72aa0570a4b', -'e38772a1730f752efeaf38cddfac1c4f', -'e39630c2c50040e463466d81e8a3d368', -'e3b485a297cf900b06de172e547a2963', -'e62d826463fa1c40cd22cbb6f64447d5', -'e63629bd5e5e84b2d8512a52fc2df31c', -'e67e35df969cc080975a0de540f77473', -'e687d2e57ef89208ad82b89b76c85ee8', -'e6f50d14a429b1de2c3dbb3276ed4b12', -'e74914315becf19950732ddb7af4b4b8', -'e777d35edb3fed07666b865f73ddd263', -'e794a44fb73b60a9a9be1faadecaff80', -'e80cde424610752721ae51b421fd84e3', -'e8146ec5487c89f71c65d610e4907edd', -'e84b60c31ac8b8b6d7b645714628448d', -'e88e2925289ee0ff74e70ac17486eecb', -'e8d35f97433156519ecfab0f1843b335', -'e8f8adeaa0c6687eb5edd1c075ae71f6', -'e93a11d1058d6a5cd683ea948bb1cc25', -'e957800efa291188263053adfee79daf', -'e9cfcd957116e2dac2c16159afb96f1b', -'ea1488f19b1cb5bb28e0f2ffd114225d', -'ea2c54df3c2bdd4396557fd31f6fee39', -'ea2ce262b4febcd5fa9060d58ae56482', -'ea70196d110d0fac0b43be584c88bdcf', -'eacd265eb9515a3c8701efb0e1e242ba', -'ead7e35a3a75f24bf619b7da0b46d872', -'eaef0f262df250febbce64c13194845c', -'eb280e53dae708501e64fbd6f13b93cd', -'eb64880ea975c14404fbb754da65eed0', -'eb68b96094324aad7f16c9b616f40b9b', -'ebc99ff869ce90e5786554dcc3c18768', -'ebea5bba51a01b80a98a0ec0e92ac4d2', -'ebf4d67bf0dd83026536f12c5fb3cb87', -'ec6419bcfac331cfe7a35e999f8a97ee', -'ec7073e291786c0c970c94a38d381b3c', -'eca247110ac1572a20d6b01072306e15', -'eccad4116321872f231ecbae1ff85b40', -'ece5f77f0fb2b2e6585ccd3ad2c6f2da', -'ed784dae74d102ff2b1c42519900cacb', -'ed7d8bca93037a310353bb18e7ff718b', -'ed953a77008266d48ca9999b35baea49', -'edac13405b9f3d29c18a5e08e797334f', -'edca87cb43d4297ac20c9a4148d8b586', -'ee0ed67d58184c8e6ca426cb6120dd9d', -'ee2c21b5ebe2249c764fb5e84fdb6a7d', -'ee8ff31530da583fc82356968242e05b', -'ee9d94a042be888ac2db90c19b562c65', -'eead54a9af3e7193e0f1d497e0b35a13', -'eeea9372a541fc8154ec6bab5e1a6fa2', -'ef1d89304232b8c14962bdd88959a781', -'ef38e6bc2183dd84ccc975172f65323d', -'ef50c396a25b46c31c48a9de38c75abd', -'ef58e4519fc67e9ebb483f6f6154a774', -'ef6ce8491618d18f2bedb7062782a30c', -'f0214c697c21336a6b69e9050f2fe4b7', -'f03b47a6d7b2217c476230f9296589e8', -'f05e129d83717aee2550880aae0bbb38', -'f07ae4d596a8e0bf41ff68482333a369', -'f09d96e8c5af5d1d840b91761f6f5cb0', -'f0e8c337ffa8f649cf2171cf1e144a27', -'f12c401eb7db03ff3a9cde5dd3d2557f', -'f18512f0ffdb4604fae2a19595e683a3', -'f19b00060a0275e2cfe8b594ddfbbbb9', -'f1c603cb7c632d7e5ae887f8730472f0', -'f1df49bf684b5792ba058ed13f8b2927', -'f234e695aa0678eaf4511f37f8411dca', -'f2d148da2c49fbee048610b9868440af', -'f30db65e8774bbfac82b75fe39055eec', -'f44dc93f68013d766ebe23419d16c54d', -'f484880135519f8870ef42acb7dfbbf3', -'f51dec0686c54df7378b948217f563cd', -'f549db117aec874e257ece05e64a2cf7', -'f5eaaf90fc11d227d3ced764889cc15a', -'f5faf947039914a0ab0c439a5aac2cfa', -'f60fa919969139e22aa551e65e85491f', -'f647ad62837f7674eef14841f250ef9f', -'f6754fe51efc5bea20a2b3745feb1494', -'f69f82c1e1ba17efa24fdb35c82f8c0c', -'f6b7c006bf639219932de105fba5cea1', -'f6b7d742b7ef73cceb75a18970353b00', -'f6fc1ff372e99f26b962ecc9c6936717', -'f787b54cf8a39b582427a95b91d4b4c9', -'f79e01143694a57c063e059df3538314', -'f7b874ecbf0a0bcd64f83fe5c667c527', -'f7c3f2a0dad1110aaeadeb3b6c330408', -'f7dcdcb11b1a1ce0eaabb1b4fa2c7393', -'f8a28e766d4fb71f2fa9afe4c582ee7b', -'f8e6364ae6c653759edac206422588e0', -'f961771c93c459c794b8f17b834bbc25', -'f98402172e16053ead75c0fd8b2b5bb5', -'f9cd1dbd92a94654ac0b0ecee907a1f2', -'fa9d0f257e55821cbeaa183a63656459', -'fb39d842736777541afde0ececd96052', -'fbf7abed7cd18ea3d56b1ee3197f19f5', -'fbfc15c6914c0e247fef67bf486e1cbd', -'fc3d99a02d8709f12d26a6625fb8be6c', -'fc869fed4c42ea44e877ad1074d42db0', -'fcb276ee2cd6115b690f9f1bfb515703', -'fcbbdece059ecac3037a2ad244695500', -'fd151026d9fc548f88a50563c2f38626', -'fd16c1c8e32e429df5f36797db2e6f89', -'fd177049f8ef56b2c08b1cf1bd52aee0', -'fd80d6ac67a16152fa44261c5a5a7a72', -'fd971db40cfd5fa2e4c940f00169c580', -'fdc97a18473abb6e1528b423fb351475', -'fdf6aa371cff16cf9c52e537dfc31400', -'fe2c001eaabb6b0f5641a1f347308e9f', -'fe6ae90c158f4801364e12e5a0cdafb0', -'fe801024805e2c672bb7d0568b43fcfb', -'fe819c5dea7fa32437f5b19c4b0b1d44', -'ff04f0641b4696aa5ae43088530219b1', -'ff136d01870d1818761c72759155b0a6', -'ff71416839616f7f0fe52612502cccc7', -'ff7617302c6625609ec0cb443b0fe647', -'ffcd87951df2bfd8bc83dc5addf723e0' + '000281ba1d5f8d068e3dee96296ff102', + '0029b39c964e3c10ec92690f1835de8d', + '003527bf683ae943e07a824786de399a', + '005481bf6fba9b3f6e43cd4dfb0fef5e', + '007128bdccbe20f77acdce11366234ee', + '0092c69f4fb681c7a0859d4bebf9f212', + '00bdc846484def25260094da28c919b7', + '0135a964f0fc8ac95fdd51af2ee3d675', + '014ab82dcf7b705e3092c60cdfe3a867', + '01885aec48c675b8c1cc5403538105e0', + '020cf7ee044d55a5a8b10b8f91caf389', + '0268f57aa44e76104dbeffb0505846d1', + '0281374a2388ef6df6b45075ae6097e2', + '02a89d48f63143fd8f0edb70a7f3915f', + '0377bc3723eabb98c2327f9646739a8c', + '039370bf32703882369bdc7e48f14a0f', + '03fe8c7fd5e80dbdb5fdacb28407f1a6', + '045687ebc08dc22d14b4464fbe935ed1', + '0516066e208b084ea10db953152a9b20', + '05239ad7cc38f0e8d3f05babf698b8bf', + '052bbad7b380d8683d85455db7221f81', + '052f17fd6a1d758cb7a058aa3a629545', + '05bc648bec7f4d0f90493cdecef2aa33', + '05e3041c0ee55331c355117c995000e4', + '060712adcfa6f7b040f701ca03a266a0', + '0628326e3e4e0fdd7699dd48831d4d05', + '069752bd3600e47e1f52ae8b3198d728', + '069cad0d46b60c05331f15dfd6e45b1f', + '070f9e2c5fc8a717cce67ee4c80472d2', + '0715b1819d8aa518a89bddb271d1db0a', + '073e4b8d5014976a94a7c86218e9413b', + '07627c8f8459308006b41d71090690eb', + '078091b23e2030e00f7f9689f4e69411', + '07c53ad26c54acdab5b6c98cf8e97122', + '07e4d25502eaf99ea34b0752cae1b225', + '08dc3a21e397c01fa375b93b2a87beae', + '092a6dd4f2986baeac98bd1849da4fc1', + '09369384ca82de49218cf5e67a14de6d', + '097f64d87922de3bf8f48b054ba844b9', + '09d1607af4f2693512d1bf4622a02fa0', + '0a16b7def7b7e0a5999236916fbfb7a3', + '0a2487d9639a1c56a9bec165e84f738c', + '0a4c0075b5d84c524054731716b54bcb', + '0a676e79756552293f8049fca16e8c7e', + '0a71230513fd2e2b7f9c048beb5ffdac', + '0a795cbbd79c99c3d1da81858fe91c6c', + '0a984427cb3fc0dcefae2bc181479dae', + '0aa080e93f6fbf0a7ac7a5b66515ae66', + '0aa6251fe58c0257846f9bc639243ac0', + '0adec962e4e2d84c8d09fa28cc6f608e', + '0aed12f5285e4b1621cff35e2e16f741', + '0b1d930aaa3ee98294757bcee48073b4', + '0b4bfb566529b7769fc41c6abbd6fa26', + '0b7a86a5e9d1d8da65fa98ac6040034d', + '0b8198767521bc2f452ef317e80545a5', + '0b9527cd7f1bb46a1d50595c4a92e8f1', + '0b99235773ee8a05f423c12d5f4e9c3d', + '0bdab840876ff65808e60cefb200420a', + '0c234361660b8ecfcca6a8bd45d3c445', + '0c2cb9e37ea01d0a2a62b213c4eb5899', + '0c6cf45b5b98b535b7232ecb0043774c', + '0c8daa545af86a8d3694c51b45989981', + '0cbea54533054458364b597c3b888f47', + '0cd387f2f672ad53e18348d095c3eda6', + '0d26639585f7217e330c6355c95e58bb', + '0d52d047264a77bea36711c8bb65d20e', + '0de5c224f5790391238a995b92583784', + '0e2c99a8b88abcc2ef3d9f67d4f037cd', + '0e85728ab43a4e98eb038818d345b638', + '0e893b31a6866bd570b45efccc0a88a7', + '0ed8d05a51532399760374d7a926a683', + '0f5f3f1f6da28983f8888b7834861970', + '0f98ca8c7d66e01e1498fc62766c4906', + '0fecab2869153e06731c08ad93e7ca59', + '109a52779f4bb1ee917a9a4e73d93b5d', + '10a2cf81bab5ffc3b6fc79fc46b953b4', + '10b57bdafa2e54c48e5b52b8dff39b40', + '10ce085f9edbadbac9a5f316f8773a4b', + '114ab357e3f25d7607b8ffb29af26996', + '114f4ac6c63eacbe1ba2f9da392a5710', + '115926e5ecb579e8855980df1d57f17a', + '116e555755894ff0332128e74ebce784', + '117d58eabdc4f5eee8cb7a70a0004962', + '118f1f337541d5b1995d1534656b08a3', + '11ac4bbe9fce147954177b20f2edccbb', + '11ba53d81683d6cf9f44d245538ba488', + '11bbeb0afbbbafea91412dbe830056d3', + '11d49a25f1600139188bc0d19defa2b6', + '11f059f1872aae766a71995194e6b5af', + '12523482232547122f2f4a56953b3893', + '12b665c75d3e04cfd4c9fa548e02cf49', + '1328643a680a06d2047fb24fe07af6a7', + '13549b1d02a8f714ca1ada2cca96b18a', + '13891a137a4dfab120a35dd33159b7f0', + '13f781d434ffb6505324cace5cdca05b', + '14914453b3ce945683148bb603040bf1', + '14ca18249f85224633dcf1b6bea3d80c', + '14eccf7e319835eb36f942fc900a6d71', + '1528b11b690e20bf60daf3dcce67e6fd', + '159924469bef2a400cdb8cbaa78df820', + '15bcd197946693d9893f1c5fa593c3f6', + '15d87d5daf7f57b2cc9dd6c51c3da985', + '15e7ff8de63317c54471fa510c5ad56b', + '15f521580593d1a0c80332b563fe7478', + '160ffe5d0ebe9d7549c2293ac0c1d613', + '161993c546c061697950c2f562664319', + '1671e4d7647fe63663497f4492a95a4a', + '167b11e11daaa25e4009500b04a838fc', + '16c94bfa77976e244a02d291129ae3c5', + '16e0f3eb679eff7cf17f709e3267cc2d', + '171b728d1133d725aa74955b0beea4d4', + '17378c4626daaa51717369704ab36a38', + '177024634db51c72ff0c83199e46edd7', + '179c38d57eb85b106dd95eba45b555fc', + '179da2793bfd142f33952884d20d21d2', + '17c73e7814a5f2d26fae015990d90159', + '17d4f2ff4a93e4644f0b9e9287f63fa7', + '18061935e6ab21f528d9dbbb36310a5f', + '180a7cf96aa37f56998a485cf22dc18e', + '1848993ceba448b05a1ff8e3a1ab427f', + '1856c914b879406212e82616cdaf8afa', + '18837494e4e25257e9d077d6374e336b', + '189a8950a57553c9d3bc75a4d8975eee', + '18e54707cb6f870fca10162d819386e6', + '196119691791be16ae0dc8cda65a2e23', + '19a1e7d12c3b6b5ad7de061d4d3050b8', + '1a37b248c0a76d5b9301b125f9294baf', + '1a678fa606dd463c7dba13cc7bf3decf', + '1a6d8da22a8baf42b7b1190b43fc16b6', + '1a85bc6c5f8780a344dc2a47df01118e', + '1ac67ebed39abe083407e98aa1c0eba1', + '1b82dc0bc393b5f8f5e4dd1ed050a194', + '1b8c2f498d2aaaf3136983ded141b5d6', + '1bb5175e4054393058019717217cf797', + '1bb909859399dcd2edf2c9ba49ba0975', + '1bca76eb57556639c84a6ff18f57d653', + '1c71b772b1b24513dbc6027589a9ab28', + '1c82862cc176f3e54ee78dc8ca857549', + '1ca233fd4979364b8488ff70abaccccb', + '1cb787a79926cce3b7cf9c4f66b34c04', + '1cd6bce4c44aef512873b29c629d1be4', + '1ce44bcdd93262a15336ad62d26c9b2f', + '1cff0213011d8f31d18859559e2b9491', + '1d165247b84f46e6772d1f659eb5a5dc', + '1d9f6266df45dfa7c38f27a1def193f0', + '1db3554a011cf458e276ef57e38795fc', + '1e725890a0bf079fb0624080eac60066', + '1fe2d0680d78f8b066d15548be3c00ea', + '201a5adf58d04945b22229c716ec49cf', + '202d8ab0948b29b35fdac71b34ce47ab', + '202faffcdefdf3f55cc5f91d1604ade4', + '203feb124dd7abe0ff5115ba0fcd7522', + '209e5d06f999bac29ef6674531061ef8', + '21037e5f5ee9bb71d6445488a7322656', + '21124d7350412c4295a1a29ec5d85d2b', + '21788fae95ad3476eb4bdf4283d00ee7', + '21ccddaf6aaa908649541a933b71adf9', + '21e67b4405b8a0a9d487587a8e6392ae', + '22236ab4c15350f2efb39074e1b49dea', + '224762278a3d732fec0679d63485cbfc', + '2252a9b2116bbe248c437daa590f8a70', + '22c2dd54f7df66c54322f1c15bb5e74b', + '22c30e00ecaa6e99b925bacd41208a86', + '2346b30adc6129a2f8ceeedf47812228', + '23ad84c84e19ec68acd0af7f26d09ebd', + '23ec40063693f430644d87808e9a6f8c', + '23f3f8d03700d08eb60cd78c2a32760c', + '2433ec87a51596eb09e3e20d281817f5', + '2446fcd4a6c5dfd13e142d5a2b257263', + '245e25886fb6015c058f02c1dc0fcc6b', + '247e276ca082c52389c581d7ef3577d2', + '24af4c6467f9f1a2ae7a1599568838c9', + '24daa96e03038c673328310e4cb5d2ee', + '24e30c967af0e7f31175d45d52be2c71', + '250f4d3b1a2d0b8fe229d4ba1c74e99e', + '2524573dfc1342a7bfdc5e357924e673', + '254915b037553c1c09f310aa29fa209f', + '254d91ecbc37546d38a28076aa20714b', + '255538ca6f6595ad001c04a717c4e546', + '256d9fd5f3e62d974ed31599f9ab0a7c', + '25adfa8c4f8f99b605d1fc77826ff4e9', + '25ba0927213be082ba9e839627156daa', + '25ff0de8b0465b991a4ca36accadd697', + '2669332932d27650b00c8a582c181650', + '267e741d756587c7a57a1fbc94b88f49', + '267eac48ce51ad1af34d0f0f236c039d', + '26af6e3202e3b53d9c3c54927837146f', + '26b9d67b06ddc7a5e01603e68ba6b226', + '27a794fcd9ac4b04397597be791b144c', + '280a4af795570a047c785ef092df04bc', + '281d20d97facbfb9e049e9cae5221441', + '2864ea89bc97d03886948cd0c1017ce2', + '28b1de8ad599137214c2b06144f7074f', + '28bc6cccd51775ebbd86134547986834', + '28caa99622bbe4b37ed2950893636c01', + '28e55710154d7a253742e5b3b8050c27', + '28efed2d1f55020051fd19f6c0b4e378', + '28f735bcf1af085c4173c72b38196658', + '29aa9f4a9658875d0e2ffefd433e6224', + '29f96d669e31f3a31ba22c6ab3db18d5', + '2a252f1d230e71a99480bc8ccfffe1b3', + '2a270f3c2588e19c8c499e1ba403a213', + '2a289a4024f786d7c21bea36324edd59', + '2a51212944401e86c82e153f18de4563', + '2a575c5cb30e7999f7437b00402e4914', + '2a59bb4ad213b2bb421d1b9017beaab8', + '2a5f39b6fbbcdd5355b7eaa18c4f75bd', + '2adc6b2cf08b799d23673e6664932307', + '2af65154930f1e05e31f1e2af4a9c2fa', + '2b2249cf15c25abe780a55f9c6b18fd0', + '2b3ba80aa86e1066a408cdf63278eb44', + '2b817101673c7a41b55ccd70682e364b', + '2bb9d4e5247b0a6a2e51f0d4e9d59306', + '2c626e08adc15d0903f7ef7d0d425580', + '2c73f77f24194f7a08c458ff95da9f89', + '2c7ef33da8500df06ad85fba7a6c52ff', + '2cddeccf128d4268f01f244ebdcb2af6', + '2cdf46f102415b53307680bd8ab3c2e8', + '2d47023b43c8e2c5ef904581aa717231', + '2d65c10c6b1a40605dfb6654bee01151', + '2d85373c539558d7e279c40b00333b71', + '2db48c4b2a0291467b30d95f46a9f2e2', + '2ea0bbea18ebf4cee93ee8a8a8d23fe2', + '2ef274269a000eb1f00881bbe6490f35', + '2f15aabe7689076793bbfeb80d79245a', + '2f30c1b9a7be022d22ce54d301065f4f', + '2f4d244dc91419d6e081ba18754e8064', + '2fa0ab4a0f301da9807fa288b9b8b1b7', + '2ff4e9c5adf1e52a26e1e6218f482340', + '2fff098dc6663c0ddbdf921bbfdef6a9', + '30039517eaf9140564b07551395d4d13', + '303fefaa15e06dcabc8adb78d6c04dd7', + '30d3cdc9facf0c6548379caf5e61bb9f', + '30fe2bdb2780818e2a539f058cb6d0b5', + '313bf32bcffb5da88d86d0ed816901e3', + '3172cad333d9e85e7e972e08f1dd3a4e', + '31c142d791ead477361dab4678386dc4', + '31ff3a4ad528e7da897d5e4d61e4f5a6', + '3223030b0ff693b90c2ca644c99deb40', + '323c8c13bacc55a9ffa52ae0754f4544', + '3261ed91de1201840519b4c8ba6e541c', + '32fc6fd1d664f514199b392e98f0637e', + '3342a225d55d3bfd1460d7d7b6235bf0', + '334d6cd24dc91fdf3c0e2dae814859da', + '3369461c36bbfc479582170fc1aba248', + '337987e09416de39c7fb1d1df21ecbed', + '33c7009b84786fa8dca3943ebe69f58f', + '33de62b9b4f5f02acf8588d0f5a358ea', + '34cc1e47f5ebf6e8ad60c859aa633428', + '3514c473a63d7c489f8c1406ebc5db49', + '35182453d758d7a1fd47b259df3200d5', + '35a793e9c9ac56d8c74352180d79812a', + '36140643574894602ba02382fe5b7217', + '36daab3778b884a95ef3c4ddd4fb7b7c', + '376f987ce35063dc19e641fe846d6bbb', + '37826389b91da6bc757f531b826d5b7d', + '37913e8e492234fb6b3727a66d77533a', + '37e1350b0bad6f9c00c5d2b897c64b02', + '384049db8d6fa59a9e1844c118a36063', + '3881f1922b2f6b0d351013213148caab', + '391a599b837ee4d3e32cda5ae8a01129', + '39287b10c6bfc48fe00be990f75e15c0', + '3976ea1c6a5b688097be9ef374f0b4a4', + '3981fe9769b1b77f871336221d1fbbb4', + '398283b1f75fb525be9c46ed82fa0581', + '398e0fdf911ab0d48ec4fdf657be6a20', + '39f127b922e43efc671ec97d9f478bf5', + '3a04c7ceea3f50c8f7fadc27eaadd772', + '3a5ea124bed49fc56a0be60585380284', + '3a6ce9f7cd7b10addd593e5b44fcfd2d', + '3aa590b6cda38578111c926fce94ec96', + '3ab4b34d1a3c4b0a600fdd618efde908', + '3b251b703a9801ce242e87b0871bfbe5', + '3b3ae6b903843a01b027eacfc222bb02', + '3b3c1e5fd01561cf82ca898ab09918e5', + '3b7566ea6df8c193dd385f10496128ee', + '3b78089b7307818b2e8c2bf9e4a96803', + '3be309dc82576272d828183255819642', + '3c5a5946d3b86a85b64a46425a55714e', + '3c8669c0263e1542c24ad6604bd57786', + '3c995482e488198b7e24d9bf0f84fcfd', + '3c9e1b23a00183a9b45fd8440b37aa4d', + '3d1bdad7dc8ad8ab4fa08defb46e62ea', + '3d1e9b80bafefd9ddd46260070a2cf6c', + '3d219f091f7b6a1ab87124adae2f7e94', + '3d877de80bb2dde6b140d9d82695e1e7', + '3e531f3bbf8bc42a18a19ea8937d33b6', + '3ebaff73c670b796d24c152d000d3ff4', + '3ecc7e5036a2933f3958298a9c632912', + '3fa33a77f79a38f09fdb340b347c6b9e', + '40179ce077d28e4b8ec6e12e2a458ade', + '406d019ab76fd33dcd94f1f4ade49309', + '40e89f6f8e739454fffaad4148c96f6f', + '40ec15cb3542296803ea4ad14ead4fee', + '4109d6e3d10bd4e1b6fd44d9d3da6565', + '41362119e969765a9a0d4b1ffc1c9c9a', + '4149d733719f3a4ac2a7d5c222ef918e', + '4157abf1fd5ff6f7cc55fcf03a289f21', + '415e7cfd3261ef8295dc72030e37272c', + '4166ec33bb88dc0f023a654305b5cd55', + '41cb36d706901224acd7aeafe805b39b', + '426b213411b191ec7ea99a4f9a0a64bd', + '429c00232869ce5d2c0b5c53b3929ceb', + '42d12fc8429ed9b609ceaf22d7ac4a47', + '42fb1b4b447a64e9d1dd43168c7e1a29', + '434c79a75d496a440115bd55e4eaa2a9', + '4357de8bee11495b084b2bf16777e2dd', + '436648b6d4da9bffd87d31172ae58c9b', + '43ec92c8a31ff71cee5266cc06891d9c', + '440f084304e4bcfde47dc4e87bf84d2e', + '44318769765abfcd37e0d0c2aa5cd7e5', + '4464b81fc15b58dc0eda6031cbac520d', + '44b32061c4d34802d75b0fd8268192fb', + '44dff0e4c5c70a0fd366c389f4746b5d', + '45324979ffbf37e84ba58fd32b103d5e', + '45389cb5fdeb019f15f5ff6e4ce83856', + '455721a761d698b3bf6a6f65ddc5afdc', + '457741cead43310e419be4c798f028ab', + '457a39e18300a9416d48f93bcba36749', + '45c727cf7a54beca386c8efb6a849f6a', + '467e2453cca71a992d0698cb2b593e0e', + '4699e376449bcda91ee0aa62df5ee61f', + '46af92472808634424ae29d8038f261f', + '46be67f57b2a7c20e239cd65901066bd', + '46dbd6995d81facadf24fa7673bfdc5f', + '46ecc8679281cbbc2af4521dbefa3657', + '472736b6d1e2b0c2335583e3bf734eea', + '47377f181b8608343384a13bfb5f9789', + '4740e51bd8b298859d54455f215f8adf', + '47bc84399c3575e0270dbe8fa0236bf8', + '47e2d15c146cc6aebadbfe6fb3f1a9a5', + '480bae5ffccbae4124285bf6276e0cf5', + '489866e361c55a874d0183470c0d72dd', + '48b8342d8f47aba1a66f52e18146f877', + '48cfe5c81bce5ad75e2562c3e9b0c82e', + '4915b7a10e36ab791f2af54888b690d7', + '4918e75b3b26ffd6111f89cce5f048d8', + '49500752f1de9219d6cd37a4fd189fc6', + '49625e618989b2160e1a71174fc5f539', + '499fd9de88d0db2869593707cba0c9bf', + '49af38131791d2dee28391da6534e5b1', + '49c39f4114a2ee21eda54a74a562bd40', + '49ce0c2a9cf277e01a2bcb36d129740e', + '49fb93a4190d73166a1287fb73db9718', + '4a27920d5831178486e182d4eca24355', + '4a29f53f1b11515179d5eeae4e975c27', + '4a2d95401099aa5c5ff4caa526da9e17', + '4a61fa295a22de8560757eeccea704b9', + '4b217a4d59cc41b06b33b25a9198a96c', + '4b7f671a97d283029ef0b60b340db4d7', + '4bbe3ca34195dbbdfe0dcdb2798bb0fe', + '4bcefd38b7e89f181e0ed93d2f88145c', + '4c8424ef265d3ac57053f79a179452ed', + '4c9897ec24670ec36e9239d4d9f8292c', + '4cb6d0b4636b96453ac7243f43082593', + '4d33f14cbf2c48c8457e5b187e70549c', + '4d575bf111e79045b4b5e60a2ab73f25', + '4d917f40dc95c5162c4479349ba35fdf', + '4dbb515302b2edcbf29b150c00aa3e82', + '4dfc00bc70c68cce265eaf340c32acc5', + '4e8b0a809caadf218591824df58e5d16', + '4e8b4f143cd24dcbdb2f34d3f8cfb759', + '4ec07cbb6bd17196442e3c80ed769619', + '4ece95f5b8e4ed9e3ee5af9151757173', + '4f4b60fb078515761dd5ae96b7981883', + '4f5a39d9cc9cce96cc89b21af5cf275a', + '50a0f4baae5ed69e7893b58f0fb93d0a', + '50afa16b0bdd5903e84499011755b5ee', + '50d0eda39a11627a7566fd931b1d7da2', + '50e2148bb145a6a22d7add40577fc655', + '50f53d47e80d9ee084dbec179bbff42d', + '51846bb8e5dbfd305866f202c6a274c7', + '51c66db758a62296a1be1527ab0be4cc', + '51ed5dd964abdb8667a3f4a2164e58e4', + '5205026f9155d0375f0fae4b64aa5c96', + '523b1a3d1f42a90fada3df3c44e91d2c', + '5248afcb1be3eb36261f3d2bd1cccb0d', + '526d38a1768970d243c7892ff3ccb6e0', + '52b22af81f73000abf1812921faccfa4', + '52ef659713b9351c5f00b4260ffdee72', + '52f007b687a54d9317e97b55c69e05e3', + '530c067d4e8d06cf4c6abb9d54284ce2', + '53301c821974eb9c8676a9a95287822d', + '5340931fa00694272ea1ed65a0b0142f', + '5357294e9f1e8137c7cbcbe93487e4c1', + '5383a0cc4f04cac5eae39bc97974fd5a', + '5399da7661e88940639467979ae1ca84', + '53f1a30bad19dbcf8c840a05077faca5', + '54113debe6196b329b52dd414c32ba8d', + '5485d6ce064e6f3ff9b77654faf3c471', + '54b166ba042e89c7484f675a370ab367', + '54b88637e43cabd1618a60fc15938ea0', + '54c8047a679d43200ef4ef39a99b0d03', + '55ad8d0bf7e02284acc6b928a918e225', + '55bcfd0805ea911d5b73a5f7e1b04640', + '55e1266f85b6139c1a3b63878a6352f9', + '562345a37c75abbfdb970fd78c700078', + '563b6b092c37a119a5a434afff399cf1', + '565de2adf4e331dea123cbbacc680d39', + '567155590c1768d6057f985ace6eab1c', + '56e5f7f5206eeadafbe9ff8a9e960416', + '56f3af901191f6edaf9de972f82ae99b', + '56f6bd72ea4de9214ac652bc434cbe6a', + '575c0cd7d6d310d287d459ceb13095dd', + '579641706b1d3df4fee1293648bd63d9', + '57b29beba489ef4debdf9dafd5c0bdc7', + '57d66d5ccc423f826fe628a57ce96397', + '580054aa2370647cfe9826ccadc4b96c', + '582c2ee88cd211bc9477a5523b27ac1b', + '5849bb14f107d51839064f81660e91c8', + '58b217dc2545dccfe31b75f1b59e6e4b', + '58f4391217544ee4b92a186a07a41795', + '5911f7eae4da6bd53a082f9ddb30d131', + '591c101d6833c440210609b18095eb2b', + '5938266faebd33436de01ff1a3ec7436', + '59472321d270a4550c9ed4bece9ffd89', + '5961c455706158ac40a34e63974c90b8', + '59bf6438bcdab6867535e31803655a17', + '5a10d97db1ad8e85c6d8f471e809492f', + '5a1db956d84af84e0c104724053d3234', + '5a925d98be7719994367cc8458c71044', + '5b68f1fab59ce019284060e2ee9af815', + '5b6b731501ad653215fb7f42bbac3890', + '5bbe967a0f721d6daa709316829ec97e', + '5be951da1e739ea740e53712d562d272', + '5bebad15261f25307a93dab063589a9e', + '5c123d7c181c4e3af1add44096af70e6', + '5ca3444c645de8ef87c4cd65d5148633', + '5ce9c8a3d34288e30d5124ce99a08eac', + '5d058a9002f585b6985edec8adb075e4', + '5d0f14c76eb1b2bcdbc1245e77c0b5a9', + '5d3d2c58ce79cc181e8dd46b39ad6992', + '5db4fb6d26d4277470ddb73dfc57ccea', + '5de1d6a94d55add2a177b345deac8df0', + '5e0a310921e2ce4065447a36dd581b91', + '5ea382d38405c2f28c85be72b9e4cfb3', + '5f198e1ef7dcc9a3eaaba6b2ba2af37e', + '5f1ca699f84fe2ff99c81fbd3f52ce1d', + '5f919c5345bfee0b3c759d7626ff9371', + '5ff6690c66a27f0b3fd2d4f85fd4fbd9', + '60253b7dcaac00bde953c510c0902691', + '602cb41449b1f7dad75677f83eb1078b', + '6062d276f90d81b84883bf42193c09b6', + '60a379d8fe9b2daf9b76d73edfcc54e9', + '613270e0f1f630b0034939bfa1058000', + '6155cfe3e56f5c5cbe39eefb2f6f4c4d', + '616d783adbcc28efd193d216ae266ead', + '6178dd680e125f4e7b25eb12dde72f4b', + '61804ccb95db616851ce33a026b8a2cb', + '61c63de6086b72c2b13ed955220ae62c', + '61c722a79c087593321e43d08401da4f', + '61cd317e99c05839abc3b53a06295e10', + '61d0e32ebedf6c0ba9536d2273a6e10c', + '61fd4070d92b8d311dd8771732df776b', + '6203813ab447f00a8f9dcfee8d55b74f', + '624df4ede61e915aa3eea6d68aa0dfb9', + '6297f545b24306330c7c0db882de6578', + '62b132077d93e516948dadd6df437960', + '62b575dfeb7c9499b93e746fa5ef6d46', + '62d065fa135b0335b9ee092fceb09d1b', + '63cad32b83072d6fd80a176c04aa28fc', + '63d6ab6cd019cdfb2f895368662a906e', + '63e62d70beec7cee6d6591d72d7405fc', + '642159ac6f0e25bc36b445d1daf7cad9', + '646f822bb5d1b11fe82af1060cda3718', + '6477363ede1c6c7bbe8ff2728fbabdb6', + '64b4667258e9eeece9c12965e0509905', + '64b53b8dd5ad4a7d6cd9c2bda0948186', + '64bf323c9f81116146d7552422f35527', + '64f5c6f882e74c27712976ef9bb592c1', + '64f6bc5e6cec9bf8c4fb694c3edba5a6', + '6501f42529aa8db0473b7b0866df53b2', + '650c267c84a05ac8f7b7167a754220ec', + '6548ddb3e3078209c18f376780c252cb', + '65ab9f08ef7103b6cfea32aa748d96ff', + '6649382d7c2d5d1b834868c5b159de25', + '66897ec932ec14c43b7156b453d9aad4', + '66978745acfa563424e1b971cb9d58c1', + '66adc632cd068063ff250b210c529f57', + '66d650bb662475c6b5cf052a3826ee9e', + '66e1ac5e2e8cf3cf445eb5da5f631ad8', + '670649bd1ed52cacf4bcca8c5a90fc0a', + '673c975a8a6250fe93f092f7383eeac1', + '673d86d168833c4d1440c9248b877b83', + '6790ab2d34cfc19461a6bf255f925597', + '679647e69a20bb6482abd1e5f2b7d931', + '67cdacac202a2d3b16c1b22733996de2', + '687fa77a15b8865371d6415d7a809889', + '687fe52b65b96cb111315a77fadbe01a', + '69143bfe7dcb9c15fca1160c3730bd52', + '6929a033e96fde056b29ef49423f9f57', + '694bee21f880e1af4bc78315cf7c84ee', + '69602a09232ca64ebc83156a4ab70b5a', + '69c0fd411df6a5340681e4157f89864d', + '6a1d9cc5886030d429ec810f31fcf65d', + '6a545e8466d45e68fc83abcee91cf843', + '6a6ff872d323e4a864cd4db863883e74', + '6a7ec4e6c8321662dba76276973827ef', + '6a879aa9fde9a859ce591d49e9b7bd02', + '6b11ccc0fc5447e69e627d01cb14dbc4', + '6bc73fa1caae19ba8bf89cd5d2aa0caf', + '6bd91c969b4cbf750d6465852617b33f', + '6c1124e6418cfcc8ec970f7d38cfb280', + '6c524f8f4db28ee0c0f8d46e94f17243', + '6c8825d743b0cfd9de0c0e03d594f667', + '6cd0a5be761684710a177d11293df197', + '6d788689f2916b02dac731ca91138634', + '6dcd61aac825cf079eb911b5645c8b4b', + '6e5688e04caace69083656ff88c5df98', + '6ed89547742e0267bf49d6f94c6c1c3f', + '6ef8f1dc83ebb978e32cabb51ec66dd5', + '6f01bd45ef491ec5492dfb0e476bdd82', + '6f2d0214628cc60a977b21a2621caad9', + '6f3bfa56066daa8a6c7d9d252f8022f9', + '7005a18a68c3c4e2ef558e0e25abc47b', + '706b4c2c7609f707a135a31503c40636', + '7088da1b6be876dc5ae19afe1d37833c', + '71c6b51b202994bc21f2ff1fa543bda8', + '71e34a571b6cda8a66a3f959d275901d', + '727e7769ca161adda9fe76fcb4539696', + '737e7418319e6a03ec7af0e6ef567024', + '738d64465529ce383debf38d7f9d63ce', + '739961a39f59d8b3895635c3ec9c3c7a', + '73b7f4b873e41a7d7f1aca5101e9ef3d', + '73dcc00247460cc29323bb0403fd2193', + '744a49bbc35874854c38f44874e25ab6', + '745fbd159940239c782f59b4bdb212b6', + '74a80bc1a5533b55acd3f004365c0dbb', + '74e92c0b5874a32d5346c9fcee48d54c', + '756113b64beaca5a229f63351275d7da', + '757c44ab06beaafea129220bc923805c', + '758d6cc8e426eb18d8359259a745c443', + '75928e20bc8a672b078e9ed15cbdb012', + '75c6da2c629e93a98bf1c421320ac78e', + '75c71fae3e36d5b1764f8f31982be985', + '75f352bb8d09891980b48eaa9c555df6', + '7643a460da7a5cc72dff8683f9261fe5', + '766c42d388e59948d6e71dc6d52a6d38', + '76c9538fb44b182305cfe06bce246aec', + '76fec598f95104a6b56dba11ef99caf6', + '770776a4692cf4ac97f54bb87db53eea', + '7744fccea1e9cea8576f0e65a3943126', + '77715ac7a4c5c96c655489de48341b67', + '777c3166cac17e462490361cdb52b981', + '7785aaca0d2e9b297eb71bde93cac0c7', + '779b465ae710c5583d4a5a89e841dc13', + '783a892e7a29f9aa23100338ef646b56', + '78bcbe0556dfcc0cd7089c18d4289387', + '78c924a5e451c517f3b2d4f1de1ca114', + '78e67353cd5fa9252b5984c7fae1632e', + '792620b6c2fdb036f14a00f506ace2f1', + '793aea3587ad8d9117d5ad3d047eea58', + '795f10876dd8e00f7625c2c2dbdcd98d', + '797cfe19dda9b2be011f7d8520dfcac7', + '79886d8ba404d4db4afbb4bba89e59ba', + '79894e986cd52dff338558f3301eced0', + '7997118bfc8baf77d9e145b331ef1296', + '7a15d318053d498451a0bca306f3e118', + '7a7ceaf45da6ec49d0a3da6d1c2a8bd6', + '7a995acf71149384c6e0e7b94efcc8ea', + '7a9d262dffbd40ec0e3563cf1ecbdaea', + '7aa009a271f3f8f12b85424edb75194d', + '7aa6cfca7192cec6090649dca8ac0a0b', + '7abaa930ccab5c391b98c3da296650ad', + '7b4fb468d023a008e297734c80e26845', + '7b91f86312f3657a3d5c2031b8d2ce05', + '7bb690b68c87d2663e4eb4fef1956b5e', + '7c58a0ef5d224e9eeb9c19605713b975', + '7c6a85e8a7d400a31cf4610e624f0042', + '7c98961612490f9a2f7b429bd78554a8', + '7cf0e55aa3f54e3185ccfe5553cce885', + '7cf7bf9a344208ddead805bca022455e', + '7d38cd67ee31bba902ed0b6a48cb871c', + '7d998d6ac8b78e36c9307dba631c0960', + '7db2c942f5416731432ab46c502fbacf', + '7db95c7d7cb96f4b4b15df7c0881c324', + '7df77a8d46e5a1de2c28440561f26b0b', + '7e1ca1be21917fba17cf0afa8286ad57', + '7e7ee6ff06a20f3c2486be9d2908eea3', + '7e83af8f970d32511937ff32196bf4a2', + '7eb8b134a49511085b7ffe38305b9fa7', + '7ed9624da51bf297e5ea42467c28be2c', + '7ee8d4be62fed20982ce793a4a064d43', + '7f0a6dcfee36dcab6dca4c30b20adeba', + '7f3aacc835e7377eb31d8a719646a62e', + '7f6d0395a0616650e782b01404f814f5', + '8006b219ec4b609a222663e05d7a17fc', + '802dd07b381ced45ea3ea7fff94c6520', + '80889327f17b337f338181db03838999', + '809012d6e0e43d75de23d4dd518818cd', + '80bb14dde7d082b3d3fb9478a4d0a1d1', + '815688a1e21e9ee9ad70eb5f3ae5a9d5', + '81d1cce893f8c09876df98a7e6c073fa', + '820f2227d09fef6916f91af9612eaedb', + '823cb21fb8fda2a217311572446a8159', + '8242ed59320fd52d2bebc8c284defac8', + '82490711b8f979a9baa35569abe31a88', + '825bae2ae84c1a11a03a3b2202f3ff21', + '826a669dd3e643deffd12cdc07b6e5b6', + '827409e108a5998219799f6d75ae3ef5', + '82b8680aea6beb9721cdc8bacab5f517', + '82e27ca093879d9c52178eca9873ed30', + '82e4842c014e236e790e2d4d98e178db', + '82ec0c79731e530f99b6fdf499b55f49', + '82f2fe481b281646944503a46775e3e6', + '83c7aaf82084d700a68da0d94f8ae1e2', + '840faf7fd3be8156c8ee4afacb9c43f8', + '8418744e36de859dfeab8baac8497170', + '84205ef9e245ea6d115ff15a50706a8d', + '844c7b90e45d94b1233901776792e1df', + '84591f5c57c3bdcf86e6deb00957d906', + '8489ead9e6e14764a936d97e97d8679c', + '8514386f4a79eec3307c3c0dbd943572', + '8544899213ffc39d7757ebdea1b3c5ff', + '8554268c701c981a2d07e39ceba09b61', + '855a1cb248b5c93e0fd56e157a27d49c', + '856fb427e0cfb74e69ce81a51f2c27bb', + '85a4bf0026a29e265cf6a8c93c904dde', + '85bd68465045078e8c8998f7b3e05d4d', + '85f86fb05282cfb13c779639eee79041', + '861687d0cb74dc7fc6bec0758253e172', + '86393e2fc8cca145ec225c1bd4784e2b', + '867d0bf6027f23a1a7b6ee7f22cb0334', + '86a1516699ad13bfa3c3630a39635724', + '86ab964396ab1b1369474f820e439aaa', + '86f2b6fa9fa6e5ce9daa568984f3bd39', + '8708eab5ef80b6aba0701784825fdd32', + '8715421a9f02732abc62a296442b0030', + '872795e5abf5b3adc08c33d4bd2450d7', + '877f285468ce60b1b4f9aa3bac7c5f17', + '87e1b58ae4a4959848283a0247866407', + '8845c2783972614f263fda4050938857', + '888139cef0fb89bf3f49258e5ef68f4c', + '88c7f35d0e01ec843a5fe6d82568408d', + '88c889b7c284807cad16b01bc70a3407', + '88f3b2721bff952eb16f0ebdab8aff34', + '89041e1d6532461ea5290ecf7e63c9e9', + '89488afcd36ac32c229dad04fbfc60f9', + '89e6ab19dfbf51aef39a36878256d937', + '8a1d2c6f04d33de4de0ff0808ef93287', + '8a4128e6e64f12ec6ef2a3e0497d6113', + '8a4a460f6d2a013dce404a896f822ef9', + '8a97a559242599bdfaa27a0a0d78450b', + '8ac77de7006b2d18146ab498477fe631', + '8ade6b421f39a52258845487cbd02485', + '8af1bdd3d07f165b0a71ce5b33a89cca', + '8b265646a5b1cb8a7313d7cbbcba3cc6', + '8b489d03c50178fe9fdf3b70a8c28761', + '8bc200b7092fa17dc68fcf92fd7ff881', + '8c0e2d497795f1b13ccc64e2d4855dbb', + '8c20be197ec6551516e8c886b0cad945', + '8c62ab14403e453316e30bc384becc55', + '8c8bf61bcb408e7ab96b5deaacf4ff97', + '8cad6ddc155bf1f52a75e6b441de0874', + '8cb7507ac9249ac82ee703346fac7de2', + '8ce6bd7e475f0272889e25e8660ebb66', + '8cf0e04fee96a4b641c0bbecae3036f7', + '8d119f3c73318c23fb9a13c74f9278a3', + '8d26165011a96624847534d6ff1e5d35', + '8d4d0ca7699a9f02311aefc59abf8560', + '8d650252168d646e8950d9f94d25c144', + '8db9640ab917733da91c1c2b7f023ad1', + '8dee5236d3ca726700ab796ffeadfc52', + '8e0d470aff1dd8ddf1d52f168ab7bae9', + '8e1304d107dc9ee1177470b1212b312e', + '8e1b12aaf505104d6174732d8bfa5fef', + '8e5cfc9a8dbe1aa81d74f6876e2bfb49', + '8fb7358018491128d5ab06a33456a337', + '902a949978baa627c8e89a69b039e16c', + '90d45b6d3c7ff6ae9c0b6fc9148873e8', + '90f402eb0a3b16d814959c614fc7c544', + '910441aef51561bd628cac1bda9c5121', + '911864b1f242417fb576abd07564ba0b', + '9136b1a098edc349a159487b6034d3dd', + '91def347ab1bcf83762acde09864d44b', + '923583c00c9d0bee9c9b110784d4810a', + '924e593b13e824c5e8b0395e7c48d3d2', + '925a9449dc67d6e0e8f5b3595f72d1de', + '92a65c0e0fd9a425a68b9e8a20fcc182', + '92d9d67ebb769dcece2a8f31339b4e72', + '930944ba677b49e6f4e2bb7774744703', + '930f6fc2222a17b6611a2b2be2a83353', + '93141fcd73b55e0774ef4265200227fa', + '93195cc5c1306454f94f7e506a302826', + '93602daa68e8715c30400b6db88116c9', + '9391e2d9febbf06d0fac4df187373971', + '9395b7f1bd3c7b642ddea6abd8b62f82', + '93a91754c07693d02c6eb9879e529137', + '93f02123e34d4bc0af11365b621dfa8a', + '941d10057afc815b397ff0669a399891', + '94ab856bfde35cc05e1b9761274ca6b2', + '955445b85f03068151dfd835d6674408', + '9580fadc098f33fd73769a81c30dd6bc', + '95b322a8fbc442a8c66ffa68973d4d57', + '960052f338d05917d374fdf010da27a7', + '965f0f52e11622d497f25deb3d2c9ceb', + '969310ebb630b2bd582e66a40778f31e', + '96a79c335d0141fba7f36f2c4d40452e', + '972a908e51bc810920b6e0f0faa03f39', + '97419977b29bcf7072fee7023a940834', + '97e4dd20338e334e174052a5290953f5', + '9839a3af3470619844045a846c6f17b0', + '98541f1ab176eff543c32aee7e3521d7', + '98a5765ef891428900492b189458f55a', + '98b3f2bbc5bec9590b126b61a657585a', + '99025f3b4978e2d75c6fba6f0bd295b2', + '990b13c991d336897b0a35ba96c11fdf', + '99674e03dd1b404963437cc4db8200cd', + '999538fbee76a7397a4dee789d5217c0', + '99c5b2749fa7290632dd67b751a43474', + '9a50f8706406632573eb5ec519371c47', + '9a5b78f794d75355d59ee7719127ddf3', + '9a791f306d3c493fa28b360d036a6965', + '9adc0a0e90fc7428bd7d78a081543d90', + '9b0153790035845f127637309a72fdfc', + '9b0f1f8d0c17aee0b49df0382958e471', + '9b2abb24e4910f3413f7fa1a098f06c6', + '9b552f036651c6a36938fb4c914bd45f', + '9ba5ab5bf0ddbc6db4aaeeb0a9a132a9', + '9bd0c13645c8b12fa8823b42877d001b', + '9c02a9e7854d7e8076dd2fa7af65e9a0', + '9c0de3d7bd3b230a2f2d749aa8fe7632', + '9c613c75a7d9e3b369c3a8a90e6fbbee', + '9c7ca3b5efe8cb272e695cb4ba4e5df9', + '9c86aa0b87c08216d5be27c09ae912ef', + '9d36b192f819354ccaac8156e2aad709', + '9d3c45f891c43f317aa049c96b3e3442', + '9d8bd93d97213864176583353b5acaf1', + '9d9c0b6a43bd5e2e847465a7e369c2ae', + '9dcd0cc82e776eb35be598efdcdfdc9f', + '9dd550e9af88a5e7592caff6474a565e', + '9e02e924712fbcc9ab8b5fbad609478e', + '9e085ff38b712db252458208881a1729', + '9e5f54988d48f5adeb352cf4944d4e61', + '9ead6071933e9bd5390902c1a9ace0e5', + '9ecfc60b471f43a90479ec74e9ee2d8a', + '9ed796b348127ffb1718f9d8a8de314c', + '9ef248745ac080ea599efe1a2fb46b4c', + '9f00306c6f7766ba105e8d220964017f', + '9f3fb30f861efc69a5960e65c7e1ea64', + '9f7e41c4665e1264e52c6edfdf396d8b', + '9fa88094b923244404b1506ad408b095', + '9fc7b291a1f6e0df908682f459651683', + '9fe2c8a78837817f221c44605f3a67be', + '9fff53b2cf73c6e958ac10207ace1473', + '9fff5845bb6f25a59dbaf6eb46ffe216', + 'a0058191941088414300116894889919', + 'a00f67500785931b01d460a76ed66978', + 'a03ad1aada4ddd805c115b4cdd4d05b3', + 'a0a85041965bf816e67114ab38d29a6e', + 'a0abed53ec7a3fedb163438e8be1c8e8', + 'a0e5b678f845bf77dfc14b554848bac1', + 'a0ec5c139564054719269fb275c7b2a4', + 'a14dc50a9445da5bcd76ec37180d3c2a', + 'a176053c9f21ca729de393429ed835dd', + 'a19c57a191b182e614a1ab3002d46a00', + 'a1aec2fd6ccb57ff4d264838bf2702e6', + 'a214aa6ee2e6c3ca82516f4115811a5e', + 'a2bae7aa69b283340714a70ca5191249', + 'a2bcb8b69acb92a3fdb7074c4db149ba', + 'a2bf9a0777517984dd0ad65cdfe5d4e6', + 'a2bfadadda18c52392b77fef8ae5824b', + 'a305fccfe55b81dfa19774a704fefe98', + 'a38a8fe6ba8cdd44241e612a4e339bf4', + 'a3a33f3ab2183f9b1ba1dc30ae2af2be', + 'a424d9f189c3e47fb7a1b179057888c2', + 'a435ed22caed17ed51c36f7e0e7a9610', + 'a44f8dfa78f7151c61157ee41010b096', + 'a4572544706239f74b177280ee98b0e8', + 'a4f218de8717713d2c8c7af1f1d43325', + 'a54bd6dc1f576c4b281e94a9538312a8', + 'a54dc2359a93e830e6d0613f15741b33', + 'a563ab6f6293403dd5e2f9dca8e7fca3', + 'a59a6e2b09ef72445deec90902ebac84', + 'a5b1f8c04ebddf13d0120b606b36e468', + 'a6d406552b5e29b93518170516595255', + 'a73408903981c30d726acf07a6712ffd', + 'a73ade7e0c839bf62cf01a9ac07071a3', + 'a74e8925ae11ec4dc13796d995e3c761', + 'a761ea482c39c292ef0cf0f78f411882', + 'a7c6e8594c4052b2364f646a13f16239', + 'a8d6dfc9d28961ea1bb0ac5d4269f89f', + 'a8fb85dc54ee1e4248c7b94f991c0ef1', + 'a925a5bfecda69217ba9e8b87c22cbc5', + 'a977b3ff47770e00ad509564e6d2af31', + 'a9b5f0fa1f8218bd9a75ae2d0c316a8c', + 'a9fbe2f744fdf0fbdd5a283b5fafb3ee', + 'aa112bd9f614a83bffe9a723f4fee949', + 'aa4d266bc1d34b8a266a23b38326802f', + 'aa4dec2e019ef612407bb3c805ea1157', + 'aa5057ebbe002f6f4b1c6df187d40903', + 'aa6691f2eab7d28f000ef94cf83175c8', + 'aa6e372b4ab9bc8d8308d753267a0664', + 'aa950028a9ba45a06b024dc6adf40d3b', + 'aadff1032abd52b873442ddec72ffb9b', + 'ab637d25fe43d54e4167cb48e7b7dbe0', + 'ab98554ae0f660d968427bb08e634c14', + 'ab98c85d25084701a8b0ebcacbefdde9', + 'abc0cbf11335d8679cb31b0edeaad57e', + 'abd2013f87814e3d939b7f4f6a959750', + 'abe3be940fb2e6d1150bdc8027184b79', + 'ac12ef0c9039ba99201430c56fca5760', + 'ac2a8597bf26093cdff8a95d38ec1657', + 'aca1a6e5ee4f52d7651d664b88dab486', + 'aced115f2620426cb05eeefd56fa48c4', + 'ad368531805ff9dcb8afa357a52506f5', + 'ada26aab0b55bc5325c398c3eb7c3658', + 'adce5f9e9686041d816d726957090690', + 'adfbbe138ebfdfcd11627dd715c6457c', + 'ae0da1761638805a9d3b2c4efa21c411', + 'ae1218d977ad1e8f865aaa4bd0588836', + 'ae3516de7256743ad9271659b00b52c2', + 'ae93620e68fcbb9e746c6d782a58f320', + 'af97ad66d2b2b12563839014e201f5d6', + 'afad69f8da5d9e40225536d15bc54799', + 'b04966d2702976041ff70af6187d05a2', + 'b0784a720da6178453b0e09c1d7cad08', + 'b08327408523735aec831c263b610e95', + 'b0afc9e3b56ba7625c74517a9ef119db', + 'b0f69fd6a03c5ed7f2ba3aecfdbfd36d', + 'b14f07fa654ecf4445bb47e76280cf5c', + 'b19e199b63ec110471db997701f5f69f', + 'b1ae7dd1af59b1a5a35ac610cfe38975', + 'b22bc3afac5b3e42ada8c2a107c3a7b3', + 'b28f9d7236b2b86cb185e78cb11d75d5', + 'b3317fff300745ebe79cce6ad4f9ffdd', + 'b33a769072368af1e5517a396cdb6f72', + 'b389513ef9b573f4fb66621d88be03f3', + 'b38b2353c63a46924d449e79d1daaec6', + 'b38cc02786e5ad09c0376d27a92bf696', + 'b3a5a3dfd425ab4ab2de05e606741ee4', + 'b3b4b56cb536a7e30002efd478ab3470', + 'b3bab6eea5975f779bf77c39e594e8f8', + 'b4363f44a7566e28b6ef90a436b82ce9', + 'b48fc8fe386636daec178d2d1e094020', + 'b4a5175ecb27089b9f6c9735bf0e4bd6', + 'b4af2297a8c78cf7e4be76b4332a7465', + 'b4c5a1d67861bee42c906c23b8b4acfd', + 'b5d201196a047162e0794600f7e44e72', + 'b5e9797476bf0a8a88799bec048ab6f6', + 'b5f64a432406a049c70c9c351a584672', + 'b6ebc5decdd4c7fd9a9b7ed8b9014f91', + 'b6f30b812fb156fbbe6e3e38fe9f8d3a', + 'b70397bed451741a08d16453ff9e4eab', + 'b704af0cac0d3493a953e4663c92bcc4', + 'b724dde01d6c35869ccf68d5aacac9c9', + 'b74d6c88441275281fe24b11e5c6fc2c', + 'b7d9ac45b117cdb3764c1a469cef560f', + 'b7f9a6bc8c7c97b0f371c757b806d60e', + 'b85ebb10b0f74d4c9714cab8ca9c3fbd', + 'b873f4c82759e30af8caddec96fec759', + 'b88ebff952bc505b1aebbe11035a509c', + 'b8c7e6fd8c05ff2a59e60d335e4c48b5', + 'b8ea27d8822ceb979ebad1bdb6f75fdc', + 'b8ee38cd503dae9d79ac6d9aed3299f0', + 'b93dbcf59d2e0f0d7d752b9e7285f484', + 'b940b555fb25743fbd9c2da67002f347', + 'b9669038c85b5e0e0cfffd26318c98c5', + 'b9e3f973b4d308e8678eb0424154ad22', + 'ba59351efe64eb952c5504eff3627367', + 'ba5bc49592c5bcde492cee85a2bd2ff4', + 'bab707b152fbf16922d8a97626a788c9', + 'bb2d9b8d7080ce4a03a91434d123dda9', + 'bb4a52d05801d7bf5892432f20488c3f', + 'bb72ee39c524fee8453c89f326319dc5', + 'bbd081417fe3cd3d08feb05c9b78943d', + 'bc35b3b8bbe0860e626314849a13939f', + 'bcbd8a9f16e3afdc602209aaa31db087', + 'bd02018461e4650522af963585610890', + 'bd1de32462eb83067201e92f07c355f7', + 'bd436cf8841b821baaa0bd3368154c27', + 'bd4a764d6deddbb1f63946cd78b8b629', + 'bd57025f1b5e4f67c547944ee28f283e', + 'bd6edf62fb0ece06740d68da4bdd847f', + 'bd8163865f0e2624385bcea221117654', + 'bdb49d6f41bc50747974bfb39a907147', + 'bdd4053d9f756a88f793e5970ae1c4c9', + 'be2e73cb632f5abc02747ecf37b664ca', + 'be33c1ba3055c5f8cdbe7d2f1794aaeb', + 'be56d693d135895f90e8d591f907238e', + 'beb3449e2b27df9eda77bf72f658fe6d', + 'bebd577054ca65b6659d830a17b4b4f7', + 'becdb68b1fbd7ef0435c2cbdcd916a3d', + 'bee71f4ce1a551fadbc14b2839f3add0', + 'bf462d40ba86abf0b878fc2ee4e9572e', + 'bf68d41b1e246b244d42d8e66a1afdae', + 'bf873fa981d4c5205bdbf1381a601f65', + 'bf8bb7ee945c95eb699d1e8ebd84913d', + 'bf8c113c1d4ffe3e494b54b527d8cee4', + 'bfc027914b444b0e2064b97ab90e0790', + 'bfcbbad7a1aad3479b9c703f7e047d37', + 'bff9378e4847a6b18473f158c5517278', + 'c0307c0f56136e43e3c46ca94802ea3b', + 'c0477075243562f57002d116366620e2', + 'c04a24e5d11b2d9739608c36aa6abd6c', + 'c05360d50fd2160cb50fc30af4071d6d', + 'c0935c7edb865554aacfe7caf65439ab', + 'c0e34351819368943bbb1806e3e8b05b', + 'c101e961a58e029596273b5e06aa385e', + 'c266fe49aea96f4f370416295eb66075', + 'c2683dd4f5705cf8b2e048d08928c76f', + 'c29e8e5b1abfd35ad853d658a051f526', + 'c2b98ce479e0067f6b195324065e3153', + 'c2ba678a8326253e76c93bbc10134e32', + 'c3820aed58d03d9b6cba57271022c3b7', + 'c384d18a53d18e1a15c6a56187bf99af', + 'c396853fc72d158d8676e6ab02ce89f3', + 'c3c91eab72d942641f8850db5b9599df', + 'c47873b0d4c8c052c4f370e4d38cc3d4', + 'c4a454dc73ddbacfa3204813f9d7dad3', + 'c4af61e47ccfcc96bc0f9ebe1f53f4b0', + 'c4c95187f4a10e9b6051dff9df6b42b8', + 'c4dc5fd4bba266fae6d4f75d2654d112', + 'c4ea1cb327db64ecc7635ee472f2cf68', + 'c5af2b94afee68d56c3e008596603c4c', + 'c5fed80fcd653b373153fb5dc0ac48b8', + 'c65c4daec0dd7ac4a71df26648ba1671', + 'c68cfb94ba8b8d21be981ce34fd11dc6', + 'c80707a675dcfe089bdbf5de048e2f1b', + 'c80d70c0335e77ae67689831169bfe27', + 'c8b9e9141ef06b1929c752caf5caae50', + 'c8ba4601368e3336e7bdb86439f2eda8', + 'c8e44d119508941687b6645736793123', + 'c8fbadba56b8674ef1dcd7628ee4aa0e', + 'c923f1176aedaba9accb908e479e7423', + 'c9bcc7c177c7c5b162ba48ffc2bf5ea4', + 'ca070d23796d6ca6423164c8897f0ec1', + 'ca0a1499e54d430ac19620ff4a1f9ba8', + 'ca34118127cbd75716923afda0222efa', + 'ca7ce47815d2f3f1599fda16bcf5043b', + 'ca88872a712dc75393b70555f014f70c', + 'ca9f16470374ac7da820a6fe1069239a', + 'caa69cdcf46194d5c52552ad5156554b', + 'cba0fafd976a15454819e12933ecf936', + 'cbdcff00c61583142b8c201b44d48273', + 'cc5246a073f0a08d56155f053910f12c', + 'cc9eeecc946e9b325c97f22f6e83930b', + 'cca7cbf3311543395d1b96780ea0e982', + 'ccc4aded16cf1e5fc3c57863ce6a70dd', + 'cd002cc133019c939468d7f524573f43', + 'cd142f615ca8c4bea7373e7de06168d5', + 'cd6c463eb77006ea4bcac4ab03a8996a', + 'cd6fb6b993c252e2e63ef2cd501072de', + 'cd9bfc6b7fe5606f4176e921f4770224', + 'cdc29ef82126350af5d336e83bdd48df', + 'ce4ea3fcebd1f30367445122d3bd31b6', + 'ce95ecbbc8b78cbef52ab86ca4888244', + 'cef2fd47dde23bfa38477243250f1c5c', + 'cf7a0f2620142e307e032e3daab72ff7', + 'd06423a5a0e7c00a97f71c5645836ba6', + 'd0bd37efeb735c0ca77ceb645235beb4', + 'd0e7e1314baeda954070777cf20cc724', + 'd0f8aa8553fd7fe6dee82dd6d670ff98', + 'd11a8c564fdd0c65ad98f9aedbc51661', + 'd12db17728205d9ba6d9c87bbebe58d3', + 'd1666b0d963c473ee075c73598be7053', + 'd1a7df95806255ee47d9ad96aa0e861b', + 'd1b31eb2c1a3c7c5750ac5915f1d9718', + 'd1bfeb98ffc67d02311455c5a3cb49b0', + 'd2189246794299fc353137bac7a59f2c', + 'd23a9506c8af058d97e344bbedb341be', + 'd24198307be9be78e5de33d8793d41bc', + 'd30282d5c07663c2a6b281317e10f7f1', + 'd33db63b8b80b51f06b12713d5f44e0f', + 'd3caeee2db9be23950fa51844e75c7ca', + 'd3ea7b0d96d5a278a93b2ddf1488ef31', + 'd40b265e06549e76a8f637422cf0835c', + 'd45268719a8aa88ae9f28fc27f07c54d', + 'd49e5340d7501f22a6fe46bed009b7be', + 'd527ebcc80cbefba3aeddb91719f3040', + 'd52e9636e57a6c4f654d076a1d4ff468', + 'd5334a6b570226cd184f5bea1a985d80', + 'd53538982298230b04403a695dcb2231', + 'd583478857fc9e12b3d538ca3b65d5d5', + 'd61233796167efb681401cdd908f0f70', + 'd627b5f50bdc0fa915d39cc32e7b760d', + 'd628ffd8aee1d8dadaf1864d295f3445', + 'd68eab15bce2dc4af3a92fa815f59336', + 'd6ada30b4cb916b077d618a73a93cf04', + 'd6cc7d0e0120a6a267adac3a05f6f287', + 'd6d08f04d065382933283a788f52f080', + 'd70da1d4fd5fe85e82e660c183da108e', + 'd7c4dc223f2a4b9817b7d30940bec64c', + 'd7de5eb5311c6220975dedf108e46a39', + 'd85b9548571a9d0edc73590660c0c18c', + 'd8af9ab641b190aeec73d211dfcbf4db', + 'd8b1e3c8d7a051c927c90ae2692b3768', + 'd8d8bfc0724c93fe35791942f69dc64e', + 'd8e20ddae73f03f16348e71884c3891d', + 'd90bdd41130bfd946f0309360a7ea05d', + 'd9109defa43b38c20ad21da38cf2c6c9', + 'd9199654e9bc0c84ccc64c81e0d3c393', + 'd960f8a4fb9d37d457e5a4b46faf98d6', + 'd9b35f8ac019ae98454b6e98c279ab13', + 'da108765256f11e4edd9314e1e494fcd', + 'da3719df7e7e02d3b78dfd10531261d4', + 'dac7a910ae791d1298a7941662ef0eeb', + 'db28722a0d1a5b0c5b1e321dfd127195', + 'db86716fff3d741d014c3b46eccb414a', + 'dc256ab79aab41f9a65bb72b9a53a154', + 'dc3b8b030492260496a8e05b7637fd4b', + 'dc90b8aba25f31d80df5c3cea85685d6', + 'dcc15b327be19bc5b50e434690d35350', + 'dd35069fb1b0b9e3e6a3195b99145330', + 'dd92bdae4ed7cb64e16f782d4d7b7767', + 'ddf1d2901866aad80dfb033e6c60e601', + 'ddf25365d3b19b83f4856fe1c8ea581c', + 'de1dafe1fb480a101d7da69608e77074', + 'de6c1c7dc6cba47e1dd3b0ad207b074b', + 'de9cd88671d4347836548fa6daecf905', + 'dec43b4d74667467c771fe7322dba942', + 'dec8b28eb5eec90f9505e0ed7064d294', + 'def4ce670fa5b300eb7b677a5f193f28', + 'df00bd33406a04fdc919a6992a0911c9', + 'df10c3db0b4a41946fe5729dee65393d', + 'df3fb9fec424fb443568c2ccdc0a7a11', + 'df659e4fce1b8f5dfba57d775013bc52', + 'df8468e5ceefe03d74e3c4fc52a93a8b', + 'dfa35a09f7e1f58e5b766576ba3cba6c', + 'dfade29220faca203263212735beb7ce', + 'dff26f8c18ef6963960f929c7426e469', + 'e022129a9694b8e72c8870552b7fb277', + 'e06500ea7c2b0a626dededdf40ffd65c', + 'e0852d277488e972e8785a65f2d13fec', + 'e10ebeb92e7a3b51f97ae1cee8315622', + 'e11428b8590c781a531a3845c81c92b7', + 'e13c3af20606152ffb2cde63161debff', + 'e16861e6f1f61807f27fbb842eaede21', + 'e17a3a67838ce6bcb3d932ee6ac70965', + 'e25bbb37db9243c70cbc562e357bd2c4', + 'e25c6e73bceb590053769855b4d7aeea', + 'e26436a21b18fd1c2f7b5106db6d36fe', + 'e288e6095032e0639f9ed72aa0570a4b', + 'e2d3c5e187ec8008ff43ccab61bc2567', + 'e38772a1730f752efeaf38cddfac1c4f', + 'e39630c2c50040e463466d81e8a3d368', + 'e3b485a297cf900b06de172e547a2963', + 'e49c3becf49f27e8e03884b4e3e0c49a', + 'e4dcd21df088a081b4350ee8287faff1', + 'e56728ffacffa0f3ce10cd80ebb80f09', + 'e62d826463fa1c40cd22cbb6f64447d5', + 'e63629bd5e5e84b2d8512a52fc2df31c', + 'e67e35df969cc080975a0de540f77473', + 'e687d2e57ef89208ad82b89b76c85ee8', + 'e6f50d14a429b1de2c3dbb3276ed4b12', + 'e74914315becf19950732ddb7af4b4b8', + 'e777d35edb3fed07666b865f73ddd263', + 'e794a44fb73b60a9a9be1faadecaff80', + 'e80cde424610752721ae51b421fd84e3', + 'e8146ec5487c89f71c65d610e4907edd', + 'e82ee6dca40947ee60e05ea600a1392f', + 'e84b60c31ac8b8b6d7b645714628448d', + 'e88e2925289ee0ff74e70ac17486eecb', + 'e8d35f97433156519ecfab0f1843b335', + 'e8f8adeaa0c6687eb5edd1c075ae71f6', + 'e90346599499fd027888748e592178c9', + 'e93380c7ef8dd1571638a3fe0b7f256e', + 'e93a11d1058d6a5cd683ea948bb1cc25', + 'e957800efa291188263053adfee79daf', + 'e9cfcd957116e2dac2c16159afb96f1b', + 'ea1488f19b1cb5bb28e0f2ffd114225d', + 'ea2c54df3c2bdd4396557fd31f6fee39', + 'ea2ce262b4febcd5fa9060d58ae56482', + 'ea70196d110d0fac0b43be584c88bdcf', + 'eac641028e227ce70195e32b7c3913a6', + 'eacd265eb9515a3c8701efb0e1e242ba', + 'ead7e35a3a75f24bf619b7da0b46d872', + 'eaef0f262df250febbce64c13194845c', + 'eb280e53dae708501e64fbd6f13b93cd', + 'eb64880ea975c14404fbb754da65eed0', + 'eb68b96094324aad7f16c9b616f40b9b', + 'ebc99ff869ce90e5786554dcc3c18768', + 'ebea5bba51a01b80a98a0ec0e92ac4d2', + 'ebf4d67bf0dd83026536f12c5fb3cb87', + 'ec3061064808a4737ee5d81ca26cbc02', + 'ec6419bcfac331cfe7a35e999f8a97ee', + 'ec7073e291786c0c970c94a38d381b3c', + 'eca247110ac1572a20d6b01072306e15', + 'eccad4116321872f231ecbae1ff85b40', + 'ece5f77f0fb2b2e6585ccd3ad2c6f2da', + 'ecf82164a76b5042456062babe6f90a8', + 'ed21e146c2ac0fa03f361fc6e5968241', + 'ed784dae74d102ff2b1c42519900cacb', + 'ed7d8bca93037a310353bb18e7ff718b', + 'ed953a77008266d48ca9999b35baea49', + 'edac13405b9f3d29c18a5e08e797334f', + 'edca87cb43d4297ac20c9a4148d8b586', + 'ee0ed67d58184c8e6ca426cb6120dd9d', + 'ee2c21b5ebe2249c764fb5e84fdb6a7d', + 'ee8ff31530da583fc82356968242e05b', + 'ee9d94a042be888ac2db90c19b562c65', + 'eead54a9af3e7193e0f1d497e0b35a13', + 'eeea9372a541fc8154ec6bab5e1a6fa2', + 'ef1d89304232b8c14962bdd88959a781', + 'ef38e6bc2183dd84ccc975172f65323d', + 'ef50c396a25b46c31c48a9de38c75abd', + 'ef58e4519fc67e9ebb483f6f6154a774', + 'ef6ce8491618d18f2bedb7062782a30c', + 'efc4ef410aaf13e107388e569f2deeaf', + 'f0214c697c21336a6b69e9050f2fe4b7', + 'f03b47a6d7b2217c476230f9296589e8', + 'f05e129d83717aee2550880aae0bbb38', + 'f07ae4d596a8e0bf41ff68482333a369', + 'f09d96e8c5af5d1d840b91761f6f5cb0', + 'f0e8c337ffa8f649cf2171cf1e144a27', + 'f12c401eb7db03ff3a9cde5dd3d2557f', + 'f13392e394f090f65e5316e24ad37468', + 'f152da260d4a6f913317335c2ab4291d', + 'f18512f0ffdb4604fae2a19595e683a3', + 'f19b00060a0275e2cfe8b594ddfbbbb9', + 'f1c603cb7c632d7e5ae887f8730472f0', + 'f1df49bf684b5792ba058ed13f8b2927', + 'f234e695aa0678eaf4511f37f8411dca', + 'f2d148da2c49fbee048610b9868440af', + 'f30db65e8774bbfac82b75fe39055eec', + 'f44dc93f68013d766ebe23419d16c54d', + 'f484880135519f8870ef42acb7dfbbf3', + 'f50a51a3cad0f70b53c7ba6ce677d586', + 'f51dec0686c54df7378b948217f563cd', + 'f549db117aec874e257ece05e64a2cf7', + 'f5eaaf90fc11d227d3ced764889cc15a', + 'f5faf947039914a0ab0c439a5aac2cfa', + 'f60fa919969139e22aa551e65e85491f', + 'f647ad62837f7674eef14841f250ef9f', + 'f6754fe51efc5bea20a2b3745feb1494', + 'f69f82c1e1ba17efa24fdb35c82f8c0c', + 'f6b7c006bf639219932de105fba5cea1', + 'f6b7d742b7ef73cceb75a18970353b00', + 'f6fc1ff372e99f26b962ecc9c6936717', + 'f787b54cf8a39b582427a95b91d4b4c9', + 'f79e01143694a57c063e059df3538314', + 'f7b874ecbf0a0bcd64f83fe5c667c527', + 'f7c3f2a0dad1110aaeadeb3b6c330408', + 'f7dcdcb11b1a1ce0eaabb1b4fa2c7393', + 'f8a28e766d4fb71f2fa9afe4c582ee7b', + 'f8e6364ae6c653759edac206422588e0', + 'f90ce9f3aacdab47305c331bca1b7288', + 'f961771c93c459c794b8f17b834bbc25', + 'f98402172e16053ead75c0fd8b2b5bb5', + 'f9cd1dbd92a94654ac0b0ecee907a1f2', + 'fa9d0f257e55821cbeaa183a63656459', + 'fb39d842736777541afde0ececd96052', + 'fbf7abed7cd18ea3d56b1ee3197f19f5', + 'fbfc15c6914c0e247fef67bf486e1cbd', + 'fc3d99a02d8709f12d26a6625fb8be6c', + 'fc56a834efc66c6c109b489eab7cb94f', + 'fc869fed4c42ea44e877ad1074d42db0', + 'fcb276ee2cd6115b690f9f1bfb515703', + 'fcbbdece059ecac3037a2ad244695500', + 'fd151026d9fc548f88a50563c2f38626', + 'fd16c1c8e32e429df5f36797db2e6f89', + 'fd177049f8ef56b2c08b1cf1bd52aee0', + 'fd80d6ac67a16152fa44261c5a5a7a72', + 'fd971db40cfd5fa2e4c940f00169c580', + 'fdc97a18473abb6e1528b423fb351475', + 'fdf6aa371cff16cf9c52e537dfc31400', + 'fe2c001eaabb6b0f5641a1f347308e9f', + 'fe61dfbffd85d9ec313049a94906d62b', + 'fe6ae90c158f4801364e12e5a0cdafb0', + 'fe801024805e2c672bb7d0568b43fcfb', + 'fe819c5dea7fa32437f5b19c4b0b1d44', + 'ff04f0641b4696aa5ae43088530219b1', + 'ff136d01870d1818761c72759155b0a6', + 'ff71416839616f7f0fe52612502cccc7', + 'ff7617302c6625609ec0cb443b0fe647', + 'ffcd87951df2bfd8bc83dc5addf723e0' ) From c884a8b25064dad6701049b2525653f73503a09a Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 29 Mar 2021 10:02:27 -0400 Subject: [PATCH 065/175] Derivatives multiprocessing procs fix, tweak timer (#143) * size is a named parameter * shorten delay before derivative runs --- etc/systemd/system/idigbio-ingestion-derivatives.timer | 4 ++-- idigbio_ingestion/mediaing/derivatives.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/etc/systemd/system/idigbio-ingestion-derivatives.timer b/etc/systemd/system/idigbio-ingestion-derivatives.timer index 54655232..5467e271 100644 --- a/etc/systemd/system/idigbio-ingestion-derivatives.timer +++ b/etc/systemd/system/idigbio-ingestion-derivatives.timer @@ -3,8 +3,8 @@ Description="Timer for idigbio-ingestion derivatives" [Timer] OnBootSec=13min -OnUnitInactiveSec=10min -RandomizedDelaySec=4m +OnUnitInactiveSec=1min +RandomizedDelaySec=15 [Install] WantedBy=timers.target diff --git a/idigbio_ingestion/mediaing/derivatives.py b/idigbio_ingestion/mediaing/derivatives.py index 3329891a..76143f13 100644 --- a/idigbio_ingestion/mediaing/derivatives.py +++ b/idigbio_ingestion/mediaing/derivatives.py @@ -53,13 +53,12 @@ def main(buckets, procs=2): buckets = ('images', 'sounds') objects = objects_for_buckets(buckets) - t1 = datetime.now() logger.info("Checking derivatives for %d objects", len(objects)) if procs > 1: apidbpool.closeall() - pool = gipcpool.Pool(procs) + pool = gipcpool.Pool(size=procs) c = ilen(pool.imap_unordered(process_objects, grouper(objects, 1000))) logger.debug("Finished %d subprocesses", c) else: From fa89d03946fac2df8223683d79b34d99314f36c0 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 29 Mar 2021 18:31:13 -0400 Subject: [PATCH 066/175] remove extraneous cStringIO import --- idigbio_ingestion/lib/dwca.py | 1 - 1 file changed, 1 deletion(-) diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index 4de18df1..54822293 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -4,7 +4,6 @@ import traceback import shutil import requests -from cStringIO import StringIO from idb.helpers.logging import idblogger, getLogger from .delimited import DelimitedFile From f79d0bb2dd20ad4fe80d7d41727e434aa5e172bc Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 29 Mar 2021 18:38:53 -0400 Subject: [PATCH 067/175] add code coverage config file for py3 --- .coveragerc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..49eefce9 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,18 @@ +# This config file is used by coverage.py (and supposedly pytest-cov, although it does not seem to). + +# The protip for generating a coverage report on idb-backend is to +# change into the top level of idb-backend and then run: +# +# $ coverage run -m pytest +# +# which will use this config file to access the sources listed below. +# +# To view the coverage report, run: +# +# $ coverage report + +[run] +source = + ./idb + ./idigbio_ingestion + ./idigbio_workers From 3b9db4529f37da82fe8f76d393a0d09b9b430a50 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 29 Mar 2021 22:26:57 -0400 Subject: [PATCH 068/175] use proper pyproj Transformer pattern The functions we were using are deprecated. Use `pyproj.Transformer.from_crs` instead. See: https://pyproj4.github.io/pyproj/stable/gotchas.html#upgrading-to-pyproj-2-from-pyproj-1 --- idb/helpers/conversions.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/idb/helpers/conversions.py b/idb/helpers/conversions.py index 677ac320..00a1a045 100644 --- a/idb/helpers/conversions.py +++ b/idb/helpers/conversions.py @@ -466,19 +466,12 @@ def geoGrabber(t, d): # convert datum to a more canonical representation (no # whitespace, all uppercase) source_datum = mangleString(datum_val) - try: - # source projection - p1 = pyproj.Proj(proj="latlon", datum=source_datum) - - # destination projection - p2 = pyproj.Proj(proj="latlon", datum="WGS84") - # do the transform - # (lon, lat) - r["geopoint"] = pyproj.transform( - p1, p2, r["geopoint"][0], r["geopoint"][1]) + try: + destination_datum = "WGS84" + transformer = pyproj.Transformer.from_crs(source_datum, destination_datum) + transformer.transform(r["geopoint"][0], r["geopoint"][1]) except: - # traceback.print_exc() # create an error flag on projection creation exception (invalid source datum) # or on transform exception (point out of bounds for source # projection) From 5d0927e606d1529eb59376d03da51a6fca85cdfa Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 29 Mar 2021 22:31:10 -0400 Subject: [PATCH 069/175] use BytesIO to construct a file-like object properly --- idigbio_ingestion/lib/dwca.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index 54822293..7e57a209 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -4,6 +4,7 @@ import traceback import shutil import requests +from io import BytesIO from idb.helpers.logging import idblogger, getLogger from .delimited import DelimitedFile @@ -55,7 +56,7 @@ def __init__(self,name="dwca.zip",skipeml=False,logname=None): try: schema_parser = etree.XMLParser(no_network=False) r = requests.get(DWC_SCHEMA_URL) - r_file_like_object = StringIO(r.content) + r_file_like_object = BytesIO(r.content) parsed = etree.parse(r_file_like_object, schema_parser) schema = etree.XMLSchema(parsed) From 0d18002d0cec597862b240ef148ea5b8571fc65e Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 30 Mar 2021 12:19:37 -0400 Subject: [PATCH 070/175] Revert "use proper pyproj Transformer pattern" This reverts commit 3b9db4529f37da82fe8f76d393a0d09b9b430a50. We do not have good test coverage around these transforms from one datum to another and a lot geo-domain knowledge is needed to write those tests. Reverting this for now, since the pyproj functions we were using are deprecated, not yet removed, and not needed to complete the python3 conversion. --- idb/helpers/conversions.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/idb/helpers/conversions.py b/idb/helpers/conversions.py index 00a1a045..677ac320 100644 --- a/idb/helpers/conversions.py +++ b/idb/helpers/conversions.py @@ -466,12 +466,19 @@ def geoGrabber(t, d): # convert datum to a more canonical representation (no # whitespace, all uppercase) source_datum = mangleString(datum_val) - try: - destination_datum = "WGS84" - transformer = pyproj.Transformer.from_crs(source_datum, destination_datum) - transformer.transform(r["geopoint"][0], r["geopoint"][1]) + # source projection + p1 = pyproj.Proj(proj="latlon", datum=source_datum) + + # destination projection + p2 = pyproj.Proj(proj="latlon", datum="WGS84") + + # do the transform + # (lon, lat) + r["geopoint"] = pyproj.transform( + p1, p2, r["geopoint"][0], r["geopoint"][1]) except: + # traceback.print_exc() # create an error flag on projection creation exception (invalid source datum) # or on transform exception (point out of bounds for source # projection) From 7f31f84f75cfbaa7893461ceeeb15ec55ef3dd30 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 30 Mar 2021 13:41:59 -0400 Subject: [PATCH 071/175] add how to view code coverage to tests README --- tests/README.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/README.md b/tests/README.md index 3ad17836..0de3276e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -204,3 +204,34 @@ ceph_server_files ``` We likely need to find a new way to refresh the test dataset. + + +## Code Coverage + +To track code coverage while running test suite: + +``` +$ coverage run -m pytest +``` + +To view the coverage report: + +``` +$ coverage report +Name Stmts Miss Cover +----------------------------------------------------------------------- +idb/__init__.py 1 0 100% +idb/annotations/__init__.py 0 0 100% +idb/annotations/apply.py 21 21 0% +idb/annotations/epandda_fetcher.py 29 29 0% +idb/annotations/loader.py 30 30 0% +idb/blacklists/__init__.py 0 0 100% +idb/blacklists/derivatives.py 1 0 100% +idb/cli.py 19 7 63% +idb/clibase.py 60 28 53% +idb/config.py 37 1 97% +idb/corrections/__init__.py 0 0 100% +idb/corrections/loader.py 30 30 0% +idb/corrections/record_corrector.py 79 79 0% +... +``` From 9db2054d347157071280659697f207b8ef4403cf Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 30 Mar 2021 13:45:24 -0400 Subject: [PATCH 072/175] use print function from __future__ --- idb/data_tables/build_taxon_index.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/idb/data_tables/build_taxon_index.py b/idb/data_tables/build_taxon_index.py index c6199db1..f65ad0d4 100644 --- a/idb/data_tables/build_taxon_index.py +++ b/idb/data_tables/build_taxon_index.py @@ -1,3 +1,4 @@ +from __future__ import print_function import os from collections import Counter @@ -65,7 +66,6 @@ def get_data(path_to_checklist): for er in e: cid = er.get("coreid") - #print e.rowtype, cid, rid if cid is not None: cid = int(cid) if cid > rid: @@ -85,9 +85,9 @@ def get_data(path_to_checklist): del r["id"] yield ("taxonnames",r) if rts["rows"] % 10000 == 0: - print rts.most_common() + print (rts.most_common()) - print rts.most_common() + print (rts.most_common()) def main(): # for r in get_data(os.path.expanduser("~/Downloads/backbone-current-sorted.zip")): From 00bded64a22a45da948a7b048be4bb7c3c892161 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 30 Mar 2021 15:53:10 -0400 Subject: [PATCH 073/175] add sample code coverage report run in python3 --- tests/sample_coverage_report.txt | 101 +++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 tests/sample_coverage_report.txt diff --git a/tests/sample_coverage_report.txt b/tests/sample_coverage_report.txt new file mode 100644 index 00000000..9ff25c3a --- /dev/null +++ b/tests/sample_coverage_report.txt @@ -0,0 +1,101 @@ +$ coverage report +Name Stmts Miss Cover +----------------------------------------------------------------------- +idb/__init__.py 1 0 100% +idb/annotations/__init__.py 0 0 100% +idb/annotations/apply.py 21 21 0% +idb/annotations/epandda_fetcher.py 29 29 0% +idb/annotations/loader.py 30 30 0% +idb/blacklists/__init__.py 0 0 100% +idb/blacklists/derivatives.py 1 0 100% +idb/cli.py 19 7 63% +idb/clibase.py 60 28 53% +idb/config.py 37 1 97% +idb/corrections/__init__.py 0 0 100% +idb/corrections/loader.py 30 30 0% +idb/corrections/record_corrector.py 79 79 0% +idb/data_api/__init__.py 53 38 28% +idb/data_api/api.py 43 6 86% +idb/data_api/common.py 48 6 88% +idb/data_api/config.py 3 0 100% +idb/data_api/v1.py 80 65 19% +idb/data_api/v2.py 91 19 79% +idb/data_api/v2_download.py 111 14 87% +idb/data_api/v2_media.py 173 24 86% +idb/data_tables/__init__.py 0 0 100% +idb/data_tables/build_taxon_index.py 63 63 0% +idb/data_tables/build_taxon_kv.py 234 234 0% +idb/data_tables/locality.py 34 34 0% +idb/data_tables/locality_data.py 21 21 0% +idb/data_tables/rights_strings.py 42 30 29% +idb/data_tables/taxon.py 35 35 0% +idb/data_tables/taxon_extended.py 45 45 0% +idb/data_tables/taxon_rank.py 24 16 33% +idb/helpers/__init__.py 18 8 56% +idb/helpers/biodiversity_socket_connector.py 82 60 27% +idb/helpers/conversions.py 502 186 63% +idb/helpers/cors.py 35 8 77% +idb/helpers/encryption.py 32 21 34% +idb/helpers/etags.py 56 11 80% +idb/helpers/fieldnames.py 45 28 38% +idb/helpers/gipcpool.py 39 26 33% +idb/helpers/idb_flask_authn.py 40 7 82% +idb/helpers/logging.py 127 46 64% +idb/helpers/media_validation.py 55 3 95% +idb/helpers/memoize.py 90 23 74% +idb/helpers/rg.py 50 1 98% +idb/helpers/signals.py 40 21 48% +idb/helpers/storage.py 132 35 73% +idb/indexing/__init__.py 101 59 42% +idb/indexing/index_from_postgres.py 166 166 0% +idb/indexing/index_helper.py 39 39 0% +idb/indexing/indexer.py 137 105 23% +idb/load_annotations.py 15 15 0% +idb/load_corrections.py 11 11 0% +idb/postgres_backend/__init__.py 22 0 100% +idb/postgres_backend/db.py 351 122 65% +idb/postgres_backend/gevent_helpers.py 145 21 86% +idb/postgres_backend/stats_db.py 16 16 0% +idb/stats/__init__.py 37 21 43% +idb/stats/collect.py 100 100 0% +idb/stats/stats-backup.py 32 32 0% +idb/stats/stats_rewriter.py 64 64 0% +idigbio_ingestion/__init__.py 0 0 100% +idigbio_ingestion/cli.py 60 60 0% +idigbio_ingestion/db_check.py 411 363 12% +idigbio_ingestion/db_rsids.py 33 33 0% +idigbio_ingestion/ds_sum_counts.py 101 101 0% +idigbio_ingestion/lib/__init__.py 0 0 100% +idigbio_ingestion/lib/delimited.py 116 96 17% +idigbio_ingestion/lib/dwca.py 151 137 9% +idigbio_ingestion/lib/eml.py 100 17 83% +idigbio_ingestion/lib/fileproxy.py 72 72 0% +idigbio_ingestion/lib/util.py 15 15 0% +idigbio_ingestion/lib/waveform.py 46 11 76% +idigbio_ingestion/lib/xmlDictTools.py 67 62 7% +idigbio_ingestion/mediaing/__init__.py 32 0 100% +idigbio_ingestion/mediaing/cli.py 51 51 0% +idigbio_ingestion/mediaing/derivatives.py 204 106 48% +idigbio_ingestion/mediaing/fetcher.py 328 150 54% +idigbio_ingestion/mediaing/migrate.py 21 21 0% +idigbio_ingestion/mediaing/updatedb.py 119 99 17% +idigbio_ingestion/recordset_cleanup.py 30 30 0% +idigbio_ingestion/update_publisher_recordset.py 308 308 0% +idigbio_ingestion/verify_ceph_objects.py 76 76 0% +idigbio_workers/__init__.py 23 8 65% +idigbio_workers/config/__init__.py 0 0 100% +idigbio_workers/config/beta.py 7 7 0% +idigbio_workers/config/prod.py 7 0 100% +idigbio_workers/config/test.py 9 9 0% +idigbio_workers/generate_static_datasets.py 34 34 0% +idigbio_workers/generate_static_datasets_index.py 92 92 0% +idigbio_workers/lib/__init__.py 0 0 100% +idigbio_workers/lib/download.py 298 95 68% +idigbio_workers/lib/identification.py 15 6 60% +idigbio_workers/lib/mailer.py 31 20 35% +idigbio_workers/lib/meta_xml.py 38 4 89% +idigbio_workers/lib/query_shim.py 93 13 86% +idigbio_workers/tasks/__init__.py 0 0 100% +idigbio_workers/tasks/download.py 68 42 38% +----------------------------------------------------------------------- +TOTAL 7042 4268 39% From f2e2eb75371701ff1d52dd1138938c17afe8f4dc Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 30 Mar 2021 15:53:57 -0400 Subject: [PATCH 074/175] print is a function in py3 --- idb/data_tables/build_taxon_kv.py | 4 ++-- idb/data_tables/locality_data.py | 4 ++-- idb/data_tables/taxon.py | 4 ++-- idb/stats/stats-backup.py | 6 +++--- idb/stats/stats_rewriter.py | 4 ++-- idigbio_ingestion/lib/fileproxy.py | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/idb/data_tables/build_taxon_kv.py b/idb/data_tables/build_taxon_kv.py index 1b455248..3db6a356 100644 --- a/idb/data_tables/build_taxon_kv.py +++ b/idb/data_tables/build_taxon_kv.py @@ -213,7 +213,7 @@ def work(t): elif cand_rank in taxon_rank.mapping: rank = taxon_rank.mapping[cand_rank] else: - print "unkown rank:", cand_rank + print ("unkown rank:", cand_rank) if rank is None: if r["dwc:scientificName"].endswith(" sp.") or r["dwc:scientificName"].endswith(" sp"): @@ -245,7 +245,7 @@ def work(t): } match, score = fuzzy_wuzzy_string_new(r["dwc:genus"], rank="genus", should=should) else: - print r + print (r) return (etag, "failout", (None, None), -1, etags[etag]) if match is not None: diff --git a/idb/data_tables/locality_data.py b/idb/data_tables/locality_data.py index ba1ea307..17566d78 100755 --- a/idb/data_tables/locality_data.py +++ b/idb/data_tables/locality_data.py @@ -2490,8 +2490,8 @@ def main(): for k,v in kl["continent"].iteritems(): if v not in real_continents and v != "None": - print k,v + print (k,v) for k,v in kl["country"].iteritems(): if v not in iso_countries and v != "None": - print k,v + print (k,v) diff --git a/idb/data_tables/taxon.py b/idb/data_tables/taxon.py index d338c9be..14386eb7 100644 --- a/idb/data_tables/taxon.py +++ b/idb/data_tables/taxon.py @@ -65,7 +65,7 @@ def get_data(path_to_checklist): # fail += 1 except: traceback.print_exc() - print r + print (r) # print len(names), len(needs_name) # total_miss = 0 @@ -125,4 +125,4 @@ def main(): for dr in get_data(os.path.expanduser("~/Downloads/backbone-current-sorted.zip")): count += 1 # print dr - print count + print (count) diff --git a/idb/stats/stats-backup.py b/idb/stats/stats-backup.py index 2a5c46e6..ab8d5b6b 100644 --- a/idb/stats/stats-backup.py +++ b/idb/stats/stats-backup.py @@ -72,7 +72,7 @@ "scroll": "5m", "query": q_body } - print t + print (t) count = 0 skip = 0 for r in elasticsearch.helpers.scan(es,**q): @@ -97,5 +97,5 @@ z.writestr(fn,json.dumps(r["_source"])) if count % 10000 == 0: - print count,skip - print count,skip \ No newline at end of file + print (count,skip) + print (count,skip) \ No newline at end of file diff --git a/idb/stats/stats_rewriter.py b/idb/stats/stats_rewriter.py index e7ff36f2..daab1e6d 100644 --- a/idb/stats/stats_rewriter.py +++ b/idb/stats/stats_rewriter.py @@ -121,13 +121,13 @@ def resume_reindex(client, source_index, target_index, query=None, target_client if h["_id"] in target_docs: stats["skip"] += 1 else: - print h + print (h) target_client.index(index=target_index,doc_type=h["_type"],body=h["_source"]) stats["done"] += 1 target_docs.add(h["_id"]) if stats["docs"] % 10000 == 0: - print stats.most_common() + print (stats.most_common()) logger.info(stats.most_common()) diff --git a/idigbio_ingestion/lib/fileproxy.py b/idigbio_ingestion/lib/fileproxy.py index 5f118395..6d5914b4 100644 --- a/idigbio_ingestion/lib/fileproxy.py +++ b/idigbio_ingestion/lib/fileproxy.py @@ -117,7 +117,7 @@ def main(): except: traceback.print_exc() x = inf.dump() - print len(x), x + print (len(x), x) if __name__ == "__main__": main() From 7f68bd4f24ba4a91a2f4a7cb215be6aa2634f389 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Fri, 2 Apr 2021 14:21:38 -0400 Subject: [PATCH 075/175] remove un-used readline function readline in dwca.py is un-used, I think. dwca.py is concerned with the archive file itself. The reading of the constituent files happens in delimited.py which has its own readline function defined. --- idigbio_ingestion/lib/dwca.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index 723c42a6..a27cc48c 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -206,9 +206,3 @@ def __init__(self,filedict,fh,logname=None): while ignoreheader > 0: self._reader.next() ignoreheader -= 1 - - def readline(self,size=None): - lineDict = {} - lineDict.update(self.defaults) - lineDict.update(super(DwcaRecordFile,self).readline(size)) - return lineDict From e00e93d96db262f0e298ba541a640d8fd158eb66 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Fri, 2 Apr 2021 15:08:07 -0400 Subject: [PATCH 076/175] add stub tests for dwca.py --- tests/idigbio_ingestion/lib/test_dwca.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/idigbio_ingestion/lib/test_dwca.py diff --git a/tests/idigbio_ingestion/lib/test_dwca.py b/tests/idigbio_ingestion/lib/test_dwca.py new file mode 100644 index 00000000..923bc21e --- /dev/null +++ b/tests/idigbio_ingestion/lib/test_dwca.py @@ -0,0 +1,19 @@ +import pytest + +@pytest.mark.skip(reason="dwca.py needs refactoring") +def test_dwca_schema_parse(): + assert(False) + +@pytest.mark.skip(reason="dwca.py needs refactoring") +def test_something_about_finding_extensions(): + assert(False) + +@pytest.mark.skip(reason="dwca.py needs refactoring") +def test_get_unescaped_fieldsTerminatedBy_from_filedict(): + assert(False) + +@pytest.mark.skip(reason="dwca.py needs refactoring") +def test_get_unescaped_fieldsEnclosedBy_from_filedict(): + assert(False) + + From 875bd68b02e96aebf5d44b99e064493c4603bf4b Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Fri, 2 Apr 2021 15:15:03 -0400 Subject: [PATCH 077/175] move lookups into functions to make testable --- idigbio_ingestion/lib/dwca.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index a27cc48c..a0293068 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -173,8 +173,10 @@ def __init__(self,filedict,fh,logname=None): rowtype = filedict["#rowType"] encoding = filedict.get("#encoding", "UTF-8") - fieldsplit = filedict["#fieldsTerminatedBy"].decode('string_escape') - fieldenc = filedict["#fieldsEnclosedBy"].decode('string_escape') + fieldsplit = get_unescaped_fieldsTerminatedBy(filedict) + #fieldsplit = filedict["#fieldsTerminatedBy"].decode('string_escape') + fieldenc = get_unescaped_fieldsEnclosedBy(filedict) + #fieldenc = filedict["#fieldsEnclosedBy"].decode('string_escape') ignoreheader = int(filedict.get("#ignoreHeaderLines","0")) self.defaults = {} @@ -206,3 +208,10 @@ def __init__(self,filedict,fh,logname=None): while ignoreheader > 0: self._reader.next() ignoreheader -= 1 + + +def get_unescaped_fieldsTerminatedBy(filedict): + return filedict["#fieldsTerminatedBy"].decode('string_escape') + +def get_unescaped_fieldsEnclosedBy(filedict): + return filedict["#fieldsEnclosedBy"].decode('string_escape') \ No newline at end of file From 03538ef94722b4f6d85bff9c07e45bec53a55422 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Fri, 2 Apr 2021 15:27:11 -0400 Subject: [PATCH 078/175] put back linesTerminatedBy and make testable --- idigbio_ingestion/lib/dwca.py | 6 ++++-- tests/idigbio_ingestion/lib/test_dwca.py | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index a0293068..f9e3eeb8 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -173,10 +173,9 @@ def __init__(self,filedict,fh,logname=None): rowtype = filedict["#rowType"] encoding = filedict.get("#encoding", "UTF-8") + linesplit = get_unescaped_linesTerminatedBy(filedict) fieldsplit = get_unescaped_fieldsTerminatedBy(filedict) - #fieldsplit = filedict["#fieldsTerminatedBy"].decode('string_escape') fieldenc = get_unescaped_fieldsEnclosedBy(filedict) - #fieldenc = filedict["#fieldsEnclosedBy"].decode('string_escape') ignoreheader = int(filedict.get("#ignoreHeaderLines","0")) self.defaults = {} @@ -210,6 +209,9 @@ def __init__(self,filedict,fh,logname=None): ignoreheader -= 1 +def get_unescaped_linesTerminatedBy(filedict): + return filedict["#linesTerminatedBy"].decode('string_escape') + def get_unescaped_fieldsTerminatedBy(filedict): return filedict["#fieldsTerminatedBy"].decode('string_escape') diff --git a/tests/idigbio_ingestion/lib/test_dwca.py b/tests/idigbio_ingestion/lib/test_dwca.py index 923bc21e..fc45e38d 100644 --- a/tests/idigbio_ingestion/lib/test_dwca.py +++ b/tests/idigbio_ingestion/lib/test_dwca.py @@ -8,6 +8,11 @@ def test_dwca_schema_parse(): def test_something_about_finding_extensions(): assert(False) + +@pytest.mark.skip(reason="dwca.py needs refactoring") +def test_get_unescaped_linesTerminatedBy_from_filedict(): + assert(False) + @pytest.mark.skip(reason="dwca.py needs refactoring") def test_get_unescaped_fieldsTerminatedBy_from_filedict(): assert(False) From c340bff438f23f65ae45abe3c54e43158d7bb926 Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Fri, 2 Apr 2021 19:44:16 +0000 Subject: [PATCH 079/175] Correcting minor typos. --- idb/data_tables/build_taxon_kv.py | 2 +- tests/{rg_benckmark.py => rg_benchmark.py} | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) rename tests/{rg_benckmark.py => rg_benchmark.py} (85%) diff --git a/idb/data_tables/build_taxon_kv.py b/idb/data_tables/build_taxon_kv.py index 3db6a356..d7c00e20 100644 --- a/idb/data_tables/build_taxon_kv.py +++ b/idb/data_tables/build_taxon_kv.py @@ -213,7 +213,7 @@ def work(t): elif cand_rank in taxon_rank.mapping: rank = taxon_rank.mapping[cand_rank] else: - print ("unkown rank:", cand_rank) + print ("unknown rank:", cand_rank) if rank is None: if r["dwc:scientificName"].endswith(" sp.") or r["dwc:scientificName"].endswith(" sp"): diff --git a/tests/rg_benckmark.py b/tests/rg_benchmark.py similarity index 85% rename from tests/rg_benckmark.py rename to tests/rg_benchmark.py index b7a60fed..6ba3aa88 100644 --- a/tests/rg_benckmark.py +++ b/tests/rg_benchmark.py @@ -5,7 +5,7 @@ t = datetime.datetime.now() rg = ReverseGeocoder() -print "INIT Time", (datetime.datetime.now() - t).total_seconds()*1000 +print("INIT Time", (datetime.datetime.now() - t).total_seconds()*1000) x = [] @@ -38,5 +38,5 @@ sm = sum(times) av = sm/len(times) -print mn, q1, q2, q3, mx -print sm, av, iqr, p95, p99 +print(mn, q1, q2, q3, mx) +print(sm, av, iqr, p95, p99) From 6c1f3b718cd64e58bb4a94e18f12c5c89911705a Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Fri, 2 Apr 2021 15:44:51 -0400 Subject: [PATCH 080/175] move lookups into functions to make dwca testable (#149) * move lookups into functions to make testable --- idigbio_ingestion/lib/dwca.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index f9e3eeb8..ce2765b7 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -208,7 +208,6 @@ def __init__(self,filedict,fh,logname=None): self._reader.next() ignoreheader -= 1 - def get_unescaped_linesTerminatedBy(filedict): return filedict["#linesTerminatedBy"].decode('string_escape') @@ -216,4 +215,5 @@ def get_unescaped_fieldsTerminatedBy(filedict): return filedict["#fieldsTerminatedBy"].decode('string_escape') def get_unescaped_fieldsEnclosedBy(filedict): - return filedict["#fieldsEnclosedBy"].decode('string_escape') \ No newline at end of file + return filedict["#fieldsEnclosedBy"].decode('string_escape') + From b07ca8f8d86733f6ce5359cf9db394672484a519 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 5 Apr 2021 14:58:10 -0400 Subject: [PATCH 081/175] add tests for getting unescaped "By" symbols (#150) --- tests/idigbio_ingestion/lib/test_dwca.py | 70 ++++++++++++++++++++---- 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/tests/idigbio_ingestion/lib/test_dwca.py b/tests/idigbio_ingestion/lib/test_dwca.py index fc45e38d..7a13eab6 100644 --- a/tests/idigbio_ingestion/lib/test_dwca.py +++ b/tests/idigbio_ingestion/lib/test_dwca.py @@ -1,24 +1,70 @@ import pytest +from copy import deepcopy +from idigbio_ingestion.lib.dwca import get_unescaped_fieldsEnclosedBy +from idigbio_ingestion.lib.dwca import get_unescaped_fieldsTerminatedBy +from idigbio_ingestion.lib.dwca import get_unescaped_linesTerminatedBy -@pytest.mark.skip(reason="dwca.py needs refactoring") -def test_dwca_schema_parse(): - assert(False) -@pytest.mark.skip(reason="dwca.py needs refactoring") -def test_something_about_finding_extensions(): - assert(False) +# sample extracted during processing of recordset c38b867b-05f3-4733-802e-d8d2d3324f84 +FILEDICT_BASE = { + 'files': {'location': 'occurrence.txt'}, + '#linesTerminatedBy': '\\n', + '#ignoreHeaderLines': '1', + '#fieldsEnclosedBy': '', + 'id': {'#index': '0'}, + 'field': [ + {'#index': '1', '#term': 'http://purl.org/dc/terms/modified'}, + {'#index': '2', '#term': 'http://rs.tdwg.org/dwc/terms/institutionCode'}, + {'#index': '3', '#term': 'http://rs.tdwg.org/dwc/terms/collectionCode'}, + {'#index': '4', '#term': 'http://rs.tdwg.org/dwc/terms/basisOfRecord'}, + {'#index': '5', '#term': 'http://rs.tdwg.org/dwc/terms/occurrenceID'}, + {'#index': '6', '#term': 'http://rs.tdwg.org/dwc/terms/catalogNumber'}, + {'#index': '7', '#term': 'http://rs.tdwg.org/dwc/terms/recordedBy'}, + {'#index': '8', '#term': 'http://rs.tdwg.org/dwc/terms/preparations'}, + {'#index': '9', '#term': 'http://rs.tdwg.org/dwc/terms/otherCatalogNumbers'}, {'#index': '10', '#term': 'http://rs.tdwg.org/dwc/terms/eventDate'}, {'#index': '11', '#term': 'http://rs.tdwg.org/dwc/terms/year'}, {'#index': '12', '#term': 'http://rs.tdwg.org/dwc/terms/month'}, {'#index': '13', '#term': 'http://rs.tdwg.org/dwc/terms/day'}, {'#index': '14', '#term': 'http://rs.tdwg.org/dwc/terms/fieldNumber'}, {'#index': '15', '#term': 'http://rs.tdwg.org/dwc/terms/eventRemarks'}, {'#index': '16', '#term': 'http://rs.tdwg.org/dwc/terms/higherGeography'}, {'#index': '17', '#term': 'http://rs.tdwg.org/dwc/terms/continent'}, {'#index': '18', '#term': 'http://rs.tdwg.org/dwc/terms/country'}, {'#index': '19', '#term': 'http://rs.tdwg.org/dwc/terms/stateProvince'}, {'#index': '20', '#term': 'http://rs.tdwg.org/dwc/terms/county'}, {'#index': '21', '#term': 'http://rs.tdwg.org/dwc/terms/locality'}, {'#index': '22', '#term': 'http://rs.tdwg.org/dwc/terms/minimumDepthInMeters'}, {'#index': '23', '#term': 'http://rs.tdwg.org/dwc/terms/maximumDepthInMeters'}, {'#index': '24', '#term': 'http://rs.tdwg.org/dwc/terms/decimalLatitude'}, {'#index': '25', '#term': 'http://rs.tdwg.org/dwc/terms/decimalLongitude'}, {'#index': '26', '#term': 'http://rs.tdwg.org/dwc/terms/geodeticDatum'}, {'#index': '27', '#term': 'http://rs.tdwg.org/dwc/terms/georeferencedDate'}, {'#index': '28', '#term': 'http://rs.tdwg.org/dwc/terms/georeferenceProtocol'}, {'#index': '29', '#term': 'http://rs.tdwg.org/dwc/terms/identificationQualifier'}, {'#index': '30', '#term': 'http://rs.tdwg.org/dwc/terms/typeStatus'}, {'#index': '31', '#term': 'http://rs.tdwg.org/dwc/terms/scientificName'}, {'#index': '32', '#term': 'http://rs.tdwg.org/dwc/terms/kingdom'}, {'#index': '33', '#term': 'http://rs.tdwg.org/dwc/terms/phylum'}, {'#index': '34', '#term': 'http://rs.tdwg.org/dwc/terms/class'}, {'#index': '35', '#term': 'http://rs.tdwg.org/dwc/terms/order'}, {'#index': '36', '#term': 'http://rs.tdwg.org/dwc/terms/family'}, {'#index': '37', '#term': 'http://rs.tdwg.org/dwc/terms/genus'}, {'#index': '38', '#term': 'http://rs.tdwg.org/dwc/terms/specificEpithet'}, {'#index': '39', '#term': 'http://rs.tdwg.org/dwc/terms/infraspecificEpithet'} + ], + '#fieldsTerminatedBy': '\\t', + '#encoding': 'UTF-8', + '#rowType': 'http://rs.tdwg.org/dwc/terms/Occurrence'} +@pytest.fixture +def filedict_fieldsEnclosedBy_nothing(): + filedict = deepcopy(FILEDICT_BASE) + return filedict + +@pytest.fixture +def filedict_fieldsEnclosedBy_escapeddoublequotes(): + filedict = deepcopy(FILEDICT_BASE) + filedict['#fieldsEnclosedBy'] = '\"' + return filedict + +@pytest.fixture +def filedict_fieldsTerminatedBy_tab(): + filedict = deepcopy(FILEDICT_BASE) + filedict['#fieldsTerminatedBy'] = '\\t' + return filedict -@pytest.mark.skip(reason="dwca.py needs refactoring") def test_get_unescaped_linesTerminatedBy_from_filedict(): - assert(False) + assert(get_unescaped_linesTerminatedBy(FILEDICT_BASE) == '\n') -@pytest.mark.skip(reason="dwca.py needs refactoring") -def test_get_unescaped_fieldsTerminatedBy_from_filedict(): - assert(False) +def test_get_unescaped_fieldsTerminatedBy_from_filedict(filedict_fieldsTerminatedBy_tab): + assert(get_unescaped_fieldsTerminatedBy(filedict_fieldsTerminatedBy_tab) == '\t') + +def test_get_unescaped_fieldsEnclosedBy_from_filedict(filedict_fieldsEnclosedBy_escapeddoublequotes): + assert(get_unescaped_fieldsEnclosedBy(filedict_fieldsEnclosedBy_escapeddoublequotes) == '"') + + +@pytest.fixture +def dwca_schema(): + # sample xml schema document goes here! + # or read from the data dir. + return "xmldoc something something" @pytest.mark.skip(reason="dwca.py needs refactoring") -def test_get_unescaped_fieldsEnclosedBy_from_filedict(): +def test_dwca_schema_parse(): assert(False) +@pytest.mark.skip(reason="dwca.py needs refactoring") +def test_something_about_finding_extensions(): + assert(False) From 1d92c5b30af791536bfbcc1c11c22200e5b280d5 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 5 Apr 2021 16:23:04 -0400 Subject: [PATCH 082/175] un-escape the "By" fields using python3-compatible method 'string_escape' is no longer available through the standard interfaces and bringing it in by importing from codecs seems more ugly than using 'unicode_escape'. The `decode` method wants a bytes-like object so we have to encode the `str` first. --- idigbio_ingestion/lib/dwca.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index ce2765b7..98e5a628 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -209,11 +209,10 @@ def __init__(self,filedict,fh,logname=None): ignoreheader -= 1 def get_unescaped_linesTerminatedBy(filedict): - return filedict["#linesTerminatedBy"].decode('string_escape') + return filedict["#linesTerminatedBy"].encode().decode('unicode_escape') def get_unescaped_fieldsTerminatedBy(filedict): - return filedict["#fieldsTerminatedBy"].decode('string_escape') + return filedict["#fieldsTerminatedBy"].encode().decode('unicode_escape') def get_unescaped_fieldsEnclosedBy(filedict): - return filedict["#fieldsEnclosedBy"].decode('string_escape') - + return filedict["#fieldsEnclosedBy"].encode().decode('unicode_escape') From 2286c6a41392e15ec162849be69bf10b710c6242 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 5 Apr 2021 20:43:38 -0400 Subject: [PATCH 083/175] provide iterator methods so we can wrap unicodecsv --- idigbio_ingestion/lib/dwca.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index 98e5a628..35fe4a9b 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -204,10 +204,18 @@ def __init__(self,filedict,fh,logname=None): fh,encoding=encoding,delimiter=fieldsplit,fieldenc=fieldenc,header=fields,rowtype=rowtype, logname=self.logger.name) + # the purpose of this is just to skip the header line while ignoreheader > 0: - self._reader.next() + next(self._reader) ignoreheader -= 1 + def __iter__(self): + return self + + def __next__(self): + return next(self._reader) + + def get_unescaped_linesTerminatedBy(filedict): return filedict["#linesTerminatedBy"].encode().decode('unicode_escape') From a90f2d2406efd8f6aa028e9a3099617e12c21ee6 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 6 Apr 2021 11:53:48 -0400 Subject: [PATCH 084/175] use supported iterator pattern --- idigbio_ingestion/lib/delimited.py | 7 +++++-- idigbio_ingestion/lib/dwca.py | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/idigbio_ingestion/lib/delimited.py b/idigbio_ingestion/lib/delimited.py index 54559f39..cbb64d47 100644 --- a/idigbio_ingestion/lib/delimited.py +++ b/idigbio_ingestion/lib/delimited.py @@ -84,7 +84,7 @@ def __init__(self, fh, encoding="utf8", delimiter=",", fieldenc="\"", header=Non cn = get_canonical_name(v) t[cn[1]] += 1 else: - headerline = self._reader.next() + headerline = next(self._reader) self.lineLength = len(headerline) self.fields = {} for k, v in enumerate(headerline): @@ -110,6 +110,9 @@ def __iter__(self): """ return self + def __next__(self): + return self.readline() + def close(self): """ Closes the internally maintained filehandle @@ -131,7 +134,7 @@ def readline(self, size=None): try: lineDict = {} # self.filehandle.snap() - lineArr = self._reader.next() + lineArr = next(self._reader) self.lineCount += 1 if self.lineLength is None: diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index 35fe4a9b..aa4527c5 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -204,16 +204,16 @@ def __init__(self,filedict,fh,logname=None): fh,encoding=encoding,delimiter=fieldsplit,fieldenc=fieldenc,header=fields,rowtype=rowtype, logname=self.logger.name) - # the purpose of this is just to skip the header line + # the purpose of this is to skip any header rows while ignoreheader > 0: - next(self._reader) + next_line = self.readline() ignoreheader -= 1 def __iter__(self): return self def __next__(self): - return next(self._reader) + return self.readline() def get_unescaped_linesTerminatedBy(filedict): From 9d3e1853c67876b387e37c709418d9ce61ce41c4 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 6 Apr 2021 13:59:16 -0400 Subject: [PATCH 085/175] use print function --- idb/helpers/etags.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/idb/helpers/etags.py b/idb/helpers/etags.py index d0bf217e..2a0c9a93 100644 --- a/idb/helpers/etags.py +++ b/idb/helpers/etags.py @@ -69,8 +69,7 @@ def objectHasher(hash_type, data, sort_arrays=False, sort_keys=True): elif data is None: pass else: - print (type(data)) + print(type(data)) - # print s h.update(s.encode("utf8")) return h.hexdigest() From 641d4bcefbc028788afa73cd78f9ad5d75f0cf5e Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 6 Apr 2021 14:01:39 -0400 Subject: [PATCH 086/175] in python3 dict.keys() replaces dict.viewkeys() --- idigbio_ingestion/db_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/idigbio_ingestion/db_check.py b/idigbio_ingestion/db_check.py index 3d175726..6a1b4aab 100644 --- a/idigbio_ingestion/db_check.py +++ b/idigbio_ingestion/db_check.py @@ -392,8 +392,8 @@ def process_subfile(rf, rsid, rs_uuid_etag, rs_id_uuid, ingest=False, db=None): seen_ids.update(ids_to_add) seen_uuids.update(uuids_to_add) - eu_set = existing_etags.viewkeys() - nu_set = seen_uuids.viewkeys() + eu_set = existing_etags.keys() + nu_set = seen_uuids.keys() deletes = len(eu_set - nu_set) From 3a9dbb0018e7c3342ddc1074401007204087cbda Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Tue, 6 Apr 2021 18:37:01 +0000 Subject: [PATCH 087/175] Rename variable for clarity. --- idigbio_ingestion/mediaing/fetcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/idigbio_ingestion/mediaing/fetcher.py b/idigbio_ingestion/mediaing/fetcher.py index cc85c839..05312cc1 100644 --- a/idigbio_ingestion/mediaing/fetcher.py +++ b/idigbio_ingestion/mediaing/fetcher.py @@ -287,8 +287,8 @@ def fetch(self): except (MissingSchema, InvalidSchema, InvalidURL) as mii: self.reason = str(mii) self.status_code = Status.UNREQUESTABLE - except ConnectionError as connectione: - self.reason = str(connectione.errno) + "{0}".format(connectione) + except ConnectionError as connection_error: + self.reason = str(connection_error.errno) + "{0}".format(connection_error) self.status_code = Status.CONNECTION_ERROR else: try: From b748a7833379e66cb9721e14f3da5935fb863dc0 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 6 Apr 2021 14:40:14 -0400 Subject: [PATCH 088/175] fix linter issue (should not be spaces between decorator and function def) --- tests/idigbio_ingestion/lib/test_eml.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/idigbio_ingestion/lib/test_eml.py b/tests/idigbio_ingestion/lib/test_eml.py index 65b8c836..cf961bca 100644 --- a/tests/idigbio_ingestion/lib/test_eml.py +++ b/tests/idigbio_ingestion/lib/test_eml.py @@ -26,8 +26,6 @@ ("vertnet_sui_verts.xml", "CC0"), # cc zero and vertnet norms #("usgs_pwrc_northamerican_bees", "No license, assume Public Domain"), # this is an html file that should not parse, currently raising an untrapped Exception ]) - - def test_intellectual_rights(eml_filename, expected_license, emlpathdir): emlfilename = emlpathdir.join(eml_filename) parsed_eml = parseEml('id_placeholder_test_suite', emlfilename) From 3faade33fd1a67c95b9665eb87d0d42802c71843 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 6 Apr 2021 14:49:14 -0400 Subject: [PATCH 089/175] replace three identical functions with one --- idigbio_ingestion/lib/dwca.py | 9 ++------- tests/idigbio_ingestion/lib/test_dwca.py | 11 +++++------ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index aa4527c5..cc4a7833 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -216,11 +216,6 @@ def __next__(self): return self.readline() -def get_unescaped_linesTerminatedBy(filedict): - return filedict["#linesTerminatedBy"].encode().decode('unicode_escape') +def get_unescaped_string(escaped_string): + return escaped_string.encode().decode('unicode_escape') -def get_unescaped_fieldsTerminatedBy(filedict): - return filedict["#fieldsTerminatedBy"].encode().decode('unicode_escape') - -def get_unescaped_fieldsEnclosedBy(filedict): - return filedict["#fieldsEnclosedBy"].encode().decode('unicode_escape') diff --git a/tests/idigbio_ingestion/lib/test_dwca.py b/tests/idigbio_ingestion/lib/test_dwca.py index 7a13eab6..6059f0a1 100644 --- a/tests/idigbio_ingestion/lib/test_dwca.py +++ b/tests/idigbio_ingestion/lib/test_dwca.py @@ -1,8 +1,7 @@ import pytest from copy import deepcopy -from idigbio_ingestion.lib.dwca import get_unescaped_fieldsEnclosedBy -from idigbio_ingestion.lib.dwca import get_unescaped_fieldsTerminatedBy -from idigbio_ingestion.lib.dwca import get_unescaped_linesTerminatedBy +from idigbio_ingestion.lib.dwca import get_unescaped_string + # sample extracted during processing of recordset c38b867b-05f3-4733-802e-d8d2d3324f84 @@ -45,13 +44,13 @@ def filedict_fieldsTerminatedBy_tab(): return filedict def test_get_unescaped_linesTerminatedBy_from_filedict(): - assert(get_unescaped_linesTerminatedBy(FILEDICT_BASE) == '\n') + assert(get_unescaped_string(FILEDICT_BASE["#linesTerminatedBy"]) == '\n') def test_get_unescaped_fieldsTerminatedBy_from_filedict(filedict_fieldsTerminatedBy_tab): - assert(get_unescaped_fieldsTerminatedBy(filedict_fieldsTerminatedBy_tab) == '\t') + assert(get_unescaped_string(filedict_fieldsTerminatedBy_tab["#fieldsTerminatedBy"]) == '\t') def test_get_unescaped_fieldsEnclosedBy_from_filedict(filedict_fieldsEnclosedBy_escapeddoublequotes): - assert(get_unescaped_fieldsEnclosedBy(filedict_fieldsEnclosedBy_escapeddoublequotes) == '"') + assert(get_unescaped_string(filedict_fieldsEnclosedBy_escapeddoublequotes["#fieldsEnclosedBy"]) == '"') @pytest.fixture From d33f3f4927da8feb9c8a9ee4d4339f551af1c189 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 6 Apr 2021 14:56:48 -0400 Subject: [PATCH 090/175] Revert "replace three identical functions with one" This reverts commit 3faade33fd1a67c95b9665eb87d0d42802c71843. In the real code these functions are called with the whole filedict, not the string, so just changing the tests isn't sufficient. --- idigbio_ingestion/lib/dwca.py | 9 +++++++-- tests/idigbio_ingestion/lib/test_dwca.py | 11 ++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index cc4a7833..aa4527c5 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -216,6 +216,11 @@ def __next__(self): return self.readline() -def get_unescaped_string(escaped_string): - return escaped_string.encode().decode('unicode_escape') +def get_unescaped_linesTerminatedBy(filedict): + return filedict["#linesTerminatedBy"].encode().decode('unicode_escape') +def get_unescaped_fieldsTerminatedBy(filedict): + return filedict["#fieldsTerminatedBy"].encode().decode('unicode_escape') + +def get_unescaped_fieldsEnclosedBy(filedict): + return filedict["#fieldsEnclosedBy"].encode().decode('unicode_escape') diff --git a/tests/idigbio_ingestion/lib/test_dwca.py b/tests/idigbio_ingestion/lib/test_dwca.py index 6059f0a1..7a13eab6 100644 --- a/tests/idigbio_ingestion/lib/test_dwca.py +++ b/tests/idigbio_ingestion/lib/test_dwca.py @@ -1,7 +1,8 @@ import pytest from copy import deepcopy -from idigbio_ingestion.lib.dwca import get_unescaped_string - +from idigbio_ingestion.lib.dwca import get_unescaped_fieldsEnclosedBy +from idigbio_ingestion.lib.dwca import get_unescaped_fieldsTerminatedBy +from idigbio_ingestion.lib.dwca import get_unescaped_linesTerminatedBy # sample extracted during processing of recordset c38b867b-05f3-4733-802e-d8d2d3324f84 @@ -44,13 +45,13 @@ def filedict_fieldsTerminatedBy_tab(): return filedict def test_get_unescaped_linesTerminatedBy_from_filedict(): - assert(get_unescaped_string(FILEDICT_BASE["#linesTerminatedBy"]) == '\n') + assert(get_unescaped_linesTerminatedBy(FILEDICT_BASE) == '\n') def test_get_unescaped_fieldsTerminatedBy_from_filedict(filedict_fieldsTerminatedBy_tab): - assert(get_unescaped_string(filedict_fieldsTerminatedBy_tab["#fieldsTerminatedBy"]) == '\t') + assert(get_unescaped_fieldsTerminatedBy(filedict_fieldsTerminatedBy_tab) == '\t') def test_get_unescaped_fieldsEnclosedBy_from_filedict(filedict_fieldsEnclosedBy_escapeddoublequotes): - assert(get_unescaped_string(filedict_fieldsEnclosedBy_escapeddoublequotes["#fieldsEnclosedBy"]) == '"') + assert(get_unescaped_fieldsEnclosedBy(filedict_fieldsEnclosedBy_escapeddoublequotes) == '"') @pytest.fixture From 41895ee2f77c32c66c7aa5b26cd106e6073e4145 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Thu, 8 Apr 2021 23:12:52 -0400 Subject: [PATCH 091/175] add help for "idb datatables" (#155) Add cli help text for idb datatables. The existing sub-commands were actually junk in terms of functionality, so added some output so they are at least marginally useful. Also a few linting cleanups. Fixes #154 --- idb/cli.py | 13 +++++++------ idb/data_tables/locality_data.py | 25 +++++++++++++++++++------ idb/data_tables/rights_strings.py | 30 +++++++++++++++++++++--------- idigbio_ingestion/cli.py | 2 +- 4 files changed, 48 insertions(+), 22 deletions(-) diff --git a/idb/cli.py b/idb/cli.py index 811af46a..daabdc0a 100644 --- a/idb/cli.py +++ b/idb/cli.py @@ -15,7 +15,7 @@ from idb import (data_api, indexing, stats) # noqa -@cli.group() +@cli.group(help="Output data tables used interally by idb-backend.") def datatables(): pass @@ -32,8 +32,9 @@ def locality_data(): from idb.data_tables.locality_data import main main() -@datatables.command(name="taxon") -@fnlogged -def taxon(): - from idb.data_tables.taxon import main - main() +### "taxon" has no useful output at the moment. +# @datatables.command(name="taxon") +# @fnlogged +# def taxon(): +# from idb.data_tables.taxon import main +# main() diff --git a/idb/data_tables/locality_data.py b/idb/data_tables/locality_data.py index 17566d78..fc6acbfd 100755 --- a/idb/data_tables/locality_data.py +++ b/idb/data_tables/locality_data.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 +from __future__ import print_function +import pprint iso_two_to_three = { "AF": "AFG", @@ -2488,10 +2490,21 @@ } def main(): - for k,v in kl["continent"].iteritems(): - if v not in real_continents and v != "None": - print (k,v) - for k,v in kl["country"].iteritems(): - if v not in iso_countries and v != "None": - print (k,v) + # TODO: Consider making this into a test. + # for k,v in kl["continent"].iteritems(): + # if v not in real_continents and v != "None": + # print k,v + + # for k,v in kl["country"].iteritems(): + # if v not in iso_countries and v != "None": + # print k,v + + pp = pprint.PrettyPrinter() + + for each in globals(): + if not each.startswith("_"): + if isinstance(globals()[each], set) or isinstance(globals()[each], dict): + print(each.join("**")) + pp.pprint(globals()[each]) + print() diff --git a/idb/data_tables/rights_strings.py b/idb/data_tables/rights_strings.py index 25f1d2fd..839600c6 100644 --- a/idb/data_tables/rights_strings.py +++ b/idb/data_tables/rights_strings.py @@ -1,4 +1,6 @@ from __future__ import print_function +import pprint + import re import traceback @@ -70,8 +72,8 @@ "CC BY-NC (Attribution-NonCommercial)": "CC4 BY-NC", "CC BY-NC (Attribution-Non-Commercial)": "CC4 BY-NC", "CC BY (Attribution)": "CC4 BY", - "Creative Commons Attribution Non Commercial (CC-BY-NC) 4.0 License" : "CC4 BY-NC", - "Public Domain (CC0 1.0)" : "CC0", + "Creative Commons Attribution Non Commercial (CC-BY-NC) 4.0 License": "CC4 BY-NC", + "Public Domain (CC0 1.0)": "CC0", "CC0 1.0 (Public-domain)": "CC0", "Public Domain": "Public Domain", "License for specimen data: Public Domain (https://creativecommons.org/publicdomain/zero/1.0/). License for media, including photographs and images of specimens: CC BY-NC-SA 3.0 License (http://creativecommons.org/licenses/by-nc-sa/3.0/us/). Users of the data are encouraged to acknowledge the source of the data if any of these records or media are used for publications, analyses, reports, or on web sites. The provider of the data, and its staff, are not responsible for damages, injury or loss due to the use of these data. Fitness of use must be determined by the user of the data.": "CC0", @@ -253,10 +255,20 @@ def get_rights_attributes(s): def main(): - for s in acceptable_licenses_trans: - picked = pick_license(s) - if acceptable_licenses_trans[s] != picked: - # print s, acceptable_licenses_trans[s], picked - print() - print (s, acceptable_licenses_trans[s]) - pick_license(s, debug=True) + # TODO: Consider making this into a test. + # for s in acceptable_licenses_trans: + # picked = pick_license(s) + # if acceptable_licenses_trans[s] != picked: + # # print s, acceptable_licenses_trans[s], picked + # print + # print s, acceptable_licenses_trans[s] + # pick_license(s, debug=True) + + pp = pprint.PrettyPrinter() + + pp.pprint("*licenses*") + pp.pprint(licenses) + + print() + pp.pprint("*acceptable_licenses_trans*") + pp.pprint(acceptable_licenses_trans) diff --git a/idigbio_ingestion/cli.py b/idigbio_ingestion/cli.py index 48002d12..8094ec4c 100644 --- a/idigbio_ingestion/cli.py +++ b/idigbio_ingestion/cli.py @@ -10,7 +10,7 @@ from .mediaing import cli as mcli # noqa ignore=F401 @cli.command(name="update-publisher-recordset", - help="") + help="Scan publishers RSS feeds, download new artifacts, and update database") @fnlogged def update_publisher_recordset(): from idigbio_ingestion.update_publisher_recordset import main From 69cc15eadab848bcb532860b59b21b4fdd55e055 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Fri, 9 Apr 2021 14:29:01 -0400 Subject: [PATCH 092/175] comment and whitespace cleanup (linting) --- idigbio_ingestion/lib/eml.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/idigbio_ingestion/lib/eml.py b/idigbio_ingestion/lib/eml.py index 3bcb34e3..2cadc64d 100644 --- a/idigbio_ingestion/lib/eml.py +++ b/idigbio_ingestion/lib/eml.py @@ -20,8 +20,8 @@ def parseEml(id, emlFilename): eml = pq(filename=emlFilename, parser='xml') - ### The eml().txt() function returns an empty string instead of None if the location does not exist in the eml - ### (if there is "no text node" according to release notes https://pypi.python.org/pypi/pyquery) + # The eml().txt() function returns an empty string instead of None if the location does not exist in the eml + # (if there is "no text node" according to release notes https://pypi.python.org/pypi/pyquery) collection = {} collection["id"] = id @@ -31,23 +31,23 @@ def parseEml(id, emlFilename): rlu = getElement(eml.root.getroot(),".//resourceLogoUrl") if rlu is not None: collection["logo_url"] = rlu.text - + collection["collection_name"] = eml("dataset > title").text() # Go until we find the first non-zero-length string (should be a collection description), collection_description_blob = "" for possible_collection_description in [ - 'dataset > abstract > para', - 'symbiota > collection > abstract > para', - 'additionalMetadata > metadata > abstract > para', - # Catch all... might literally catch any other desc text anywhere in the document. - 'abstract > para' - ]: + 'dataset > abstract > para', + 'symbiota > collection > abstract > para', + 'additionalMetadata > metadata > abstract > para', + # Catch all... might literally catch any other desc text anywhere in the document. + 'abstract > para' + ]: collection_description_blob += eml.find(possible_collection_description).text() if len(collection_description_blob) > 0: break collection["collection_description"] = collection_description_blob - + iwa = getElement(eml.root.getroot(),"additionalMetadata/metadata/symbiota/collection/onlineUrl") if iwa is not None: collection["institution_web_address"] = iwa.text @@ -121,7 +121,7 @@ def parseEml(id, emlFilename): else: seen_emails.append(contact["email"]) elif ccc.tag == "positionName": - contact["role"] = ccc.text + contact["role"] = ccc.text else: if cc.text != "": if cc.tag == "individualName": @@ -141,7 +141,6 @@ def parseEml(id, emlFilename): if len(contact.keys()) > 0: collection["contacts"].append(contact) - # collection["contacts"] = [{ "name": clm.groups()[0], "email": clm.groups()[1] + "@" + clm.groups()[2]}] collection["other_guids"] = [] for g in eml("alternateidentifier"): collection["other_guids"].append(g.text) @@ -151,7 +150,7 @@ def parseEml(id, emlFilename): def main(): import sys import json - print (json.dumps(parseEml("testid",sys.argv[1]))) + print(json.dumps(parseEml("testid",sys.argv[1]))) if __name__ == '__main__': From 4b2412ee69e1702277a1c97d00e7005ef4f57188 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Fri, 9 Apr 2021 18:36:12 -0400 Subject: [PATCH 093/175] allow library to open file by passing the filename --- idigbio_ingestion/update_publisher_recordset.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/idigbio_ingestion/update_publisher_recordset.py b/idigbio_ingestion/update_publisher_recordset.py index dbdf7513..c055661b 100644 --- a/idigbio_ingestion/update_publisher_recordset.py +++ b/idigbio_ingestion/update_publisher_recordset.py @@ -436,8 +436,7 @@ def harvest_eml(r, db): u, _, _ = db.get_uuid(r["recordids"]) logger.debug("Using recordset UUID: {0}".format(u)) desc = {} - with open(fname,"rb") as inf: - desc = parseEml(r["recordids"][0], inf.read()) + desc = parseEml(r["recordids"][0], fname) desc["ingest"] = r["ingest"] desc["link"] = r["file_link"] desc["eml_link"] = r["eml_link"] From 1396098779fdb461400b73977eae7153e8b16c4f Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 11 Apr 2021 08:14:53 -0400 Subject: [PATCH 094/175] urlparse library renamed to urllib.parse in python3 --- idb/indexing/index_helper.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/idb/indexing/index_helper.py b/idb/indexing/index_helper.py index f8cf1eaa..c2e7c65e 100644 --- a/idb/indexing/index_helper.py +++ b/idb/indexing/index_helper.py @@ -9,8 +9,7 @@ from idb.helpers.logging import idblogger logger = idblogger.getChild('index_helper') -# PYTHON3_WARNING -from urlparse import urlparse +from urllib.parse import urlparse def index_record(ei, rc, typ, r, do_index=True): """ From e1352affd865ba21e528e9fdd81418f093b2c0fa Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 11 Apr 2021 09:34:25 -0400 Subject: [PATCH 095/175] whitespace and garbage comment cleanup --- idb/corrections/record_corrector.py | 6 +++--- idb/data_tables/build_taxon_kv.py | 2 +- idb/data_tables/taxon_rank.py | 2 +- idb/indexing/index_helper.py | 2 +- idb/indexing/indexer.py | 2 +- idb/postgres_backend/db.py | 2 +- idb/stats/collect.py | 2 +- idb/stats/stats_rewriter.py | 14 +++++++------- idigbio_ingestion/db_check.py | 6 +++--- idigbio_ingestion/lib/fileproxy.py | 8 ++++---- idigbio_ingestion/lib/xmlDictTools.py | 2 +- 11 files changed, 24 insertions(+), 24 deletions(-) diff --git a/idb/corrections/record_corrector.py b/idb/corrections/record_corrector.py index c8887f48..3a570bf8 100644 --- a/idb/corrections/record_corrector.py +++ b/idb/corrections/record_corrector.py @@ -95,15 +95,15 @@ def get_etag(t): # apply a flag instead. # print(t, self.corrections[etag]) # consider adding logging / debug lines instead - + if ( "dwc:kingdom" in self.corrections[etag] and "dwc:kingdom" in corrected_dict and corrected_dict["dwc:kingdom"].lower() != self.corrections[etag]["dwc:kingdom"] and corrected_dict["dwc:kingdom"].lower() in protected_kingdoms ): - corrected_dict["flag_dwc_kingdom_suspect"] = True - continue + corrected_dict["flag_dwc_kingdom_suspect"] = True + continue for k in self.corrections[etag].keys(): if k == "dwc:scientificname": diff --git a/idb/data_tables/build_taxon_kv.py b/idb/data_tables/build_taxon_kv.py index d7c00e20..5a6fbf1b 100644 --- a/idb/data_tables/build_taxon_kv.py +++ b/idb/data_tables/build_taxon_kv.py @@ -292,7 +292,7 @@ def get_taxon_from_index(): etag = objectHasher("sha256", r["_source"]["data"], sort_arrays=True) stats["count"] += 1 if etag not in etags: - stats["precount"] += 1 + stats["precount"] += 1 try: yield (etag,r["_source"]["data"]) except KeyError as e: diff --git a/idb/data_tables/taxon_rank.py b/idb/data_tables/taxon_rank.py index 0a965d2e..2d5a725f 100644 --- a/idb/data_tables/taxon_rank.py +++ b/idb/data_tables/taxon_rank.py @@ -389,6 +389,6 @@ def main(): print (json.dumps(list(new_accept_set), indent=4)) - + if __name__ == '__main__': main() diff --git a/idb/indexing/index_helper.py b/idb/indexing/index_helper.py index c2e7c65e..74fe64ce 100644 --- a/idb/indexing/index_helper.py +++ b/idb/indexing/index_helper.py @@ -70,7 +70,7 @@ def index_record(ei, rc, typ, r, do_index=True): i["data"] = r["data"] i["indexData"] = d - if config.IDB_EXTRA_SERIOUS_DEBUG == 'yes': # dummy comment because github + if config.IDB_EXTRA_SERIOUS_DEBUG == 'yes': logger.debug("Index record: %s with approx. %s bytes of data.", i["uuid"], len(repr(i))) logger.debug("Data: %s", repr(i)) diff --git a/idb/indexing/indexer.py b/idb/indexing/indexer.py index 4f786f92..44788316 100644 --- a/idb/indexing/indexer.py +++ b/idb/indexing/indexer.py @@ -128,7 +128,7 @@ def __init__(self, indexName, types, } # Create index only if: - # 1. it does not exist, and + # 1. it does not exist, and # 2. we have the environment variable set to permit index creation. self.ALLOW_INDEX_CREATION = True if config.ES_ALLOW_INDEX_CREATION == "yes" else False if not self.ALLOW_INDEX_CREATION and not self.es.indices.exists(index=self.indexName): diff --git a/idb/postgres_backend/db.py b/idb/postgres_backend/db.py index d7f3c545..4753f41c 100644 --- a/idb/postgres_backend/db.py +++ b/idb/postgres_backend/db.py @@ -77,7 +77,7 @@ class PostgresDB(object): """ + \ __join_uuids_etags_latest_version + \ __join_uuids_identifiers + \ - __join_uuids_siblings + __join_uuids_siblings __columns_master_query = """ SELECT uuids.id as uuid, diff --git a/idb/stats/collect.py b/idb/stats/collect.py index e167e159..17146221 100644 --- a/idb/stats/collect.py +++ b/idb/stats/collect.py @@ -113,7 +113,7 @@ def collect_stats(collect_datetime, es=None): recordset_stats = defaultdict(new_stats_dict) - # print date_min, date_max + sql = """SELECT * FROM stats LEFT JOIN queries on stats.query_id=queries.id WHERE date > %s AND date < %s diff --git a/idb/stats/stats_rewriter.py b/idb/stats/stats_rewriter.py index daab1e6d..5a4e502b 100644 --- a/idb/stats/stats_rewriter.py +++ b/idb/stats/stats_rewriter.py @@ -57,7 +57,7 @@ def resume_reindex(client, source_index, target_index, query=None, target_client size=5000 ): target_docs.add(h['_id']) - logger.info("Skip list build: {} docs".format(len(target_docs))) + logger.info("Skip list build: {} docs".format(len(target_docs))) base_num = 0 end_num = 48 @@ -122,7 +122,7 @@ def resume_reindex(client, source_index, target_index, query=None, target_client stats["skip"] += 1 else: print (h) - target_client.index(index=target_index,doc_type=h["_type"],body=h["_source"]) + target_client.index(index=target_index,doc_type=h["_type"],body=h["_source"]) stats["done"] += 1 target_docs.add(h["_id"]) @@ -330,12 +330,12 @@ def resume_reindex(client, source_index, target_index, query=None, target_client # m = es.indices.get_mapping(index=source_index,doc_type="api,digest,fields,search") # m[source_index]["mappings"]["api"]["properties"]["recordset_id"] ={ -# "type": "string", +# "type": "string", # "analyzer": "keyword" # } # m[source_index]["mappings"]["digest"]["properties"]["recordset_id"] ={ -# "type": "string", +# "type": "string", # "analyzer": "keyword" # } @@ -374,12 +374,12 @@ def resume_reindex(client, source_index, target_index, query=None, target_client # del dk[t]["properties"][k] # m[source_index]["mappings"]["api"]["properties"]["recordset_id"] ={ -# "type": "string", +# "type": "string", # "analyzer": "keyword" # } # m[source_index]["mappings"]["digest"]["properties"]["recordset_id"] ={ -# "type": "string", +# "type": "string", # "analyzer": "keyword" # } @@ -398,7 +398,7 @@ def resume_reindex(client, source_index, target_index, query=None, target_client # data["keys"][t][re_map[k]] = data["keys"][t][k] # else: # data["keys"][t][re_map[k]] += data["keys"][t][k] -# del data["keys"][t][k] +# del data["keys"][t][k] # return (action, data) recordsets = [ diff --git a/idigbio_ingestion/db_check.py b/idigbio_ingestion/db_check.py index 6a1b4aab..88198ca7 100644 --- a/idigbio_ingestion/db_check.py +++ b/idigbio_ingestion/db_check.py @@ -276,7 +276,7 @@ def process_subfile(rf, rsid, rs_uuid_etag, rs_id_uuid, ingest=False, db=None): u, parent, deleted = db.get_uuid([i for _,_,i in idents]) if parent is not None: if parent != rsid: - if config.IDB_EXTRA_SERIOUS_DEBUG == 'yes': + if config.IDB_EXTRA_SERIOUS_DEBUG == 'yes': rlogger.debug("******") rlogger.debug("u: {0}".format(u)) rlogger.debug("parent: {0}".format(parent)) @@ -359,7 +359,7 @@ def process_subfile(rf, rsid, rs_uuid_etag, rs_id_uuid, ingest=False, db=None): db_uuid = db_r["uuid"] if db_uuid is None: - # We probably need to do something other than raise this RecordException for this since + # We probably need to do something other than raise this RecordException for this since # ac:associatedSpecimenReference is not required to be in *this* file / system. # Can we find a different identifier? raise RecordException("Record (idents: [{0}]) contains ac:associatedSpecimenReference '{1}' that does not relate to an existing identifier.".format(ref_uuid, idents)) @@ -545,7 +545,7 @@ def metadataToSummaryJSON(rsid, metadata, writeFile=True, doStats=True): with AtomicFile(rsid + ".summary.json", "w") as jf: json.dump(summary, jf, indent=2) return summary - + csv_line_count = 0 no_recordid_count = 0 diff --git a/idigbio_ingestion/lib/fileproxy.py b/idigbio_ingestion/lib/fileproxy.py index 6d5914b4..99dae8f6 100644 --- a/idigbio_ingestion/lib/fileproxy.py +++ b/idigbio_ingestion/lib/fileproxy.py @@ -21,7 +21,7 @@ class FileProxy(object): """ - The main FileProxy class, behaves just like a file object, with three additions: + The main FileProxy class, behaves just like a file object, with three additions: * dump() is added to return a string context of all bytes read since the last snapshot. * snap() is added to take a snapshot of the number of bytes read so far. * next() and read() are modified to capture the number of bytes read from the file. @@ -42,12 +42,12 @@ def dump(self): save = self.__file.tell() self.__file.seek(max(self.__snapshot - self.__add, 0)) s = self.__file.read(self.__read_chars - self.__snapshot + (self.__add * 2)) - self.__file.seek(save) + self.__file.seek(save) return s def close(self): return self.__file.close() - + def flush(self): return self.__file.flush() @@ -69,7 +69,7 @@ def read(self,size=None): def readline(self,size=None): return self.__file.readline(size) - + def readlines(self,sizehint=None): return self.__file.readlines(sizehint) diff --git a/idigbio_ingestion/lib/xmlDictTools.py b/idigbio_ingestion/lib/xmlDictTools.py index 0a7be1bc..405ab6a2 100755 --- a/idigbio_ingestion/lib/xmlDictTools.py +++ b/idigbio_ingestion/lib/xmlDictTools.py @@ -92,7 +92,7 @@ def d2xml(d): def _d2xml(d, p): for k,v in d.items(): if k.startswith("#"): - p.set(k[1:],v) + p.set(k[1:],v) elif isinstance(v,dict): node = etree.SubElement(p, k) _d2xml(v, node) From 744e54b713fa2020a42a046ede5db1b4f7b273a4 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 11 Apr 2021 11:18:42 -0400 Subject: [PATCH 096/175] declare string for regex as raw string to allow backslash --- idb/helpers/rg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idb/helpers/rg.py b/idb/helpers/rg.py index 0f6a72e4..f736e5f8 100644 --- a/idb/helpers/rg.py +++ b/idb/helpers/rg.py @@ -9,7 +9,7 @@ from .memoize import memoized -pattern = re.compile('[\W_]+') +pattern = re.compile(r'[\W_]+') class ReverseGeocoder(object): From 5ece3dbb71dd358a594038429e6ed5b38c054b19 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 11 Apr 2021 13:11:40 -0400 Subject: [PATCH 097/175] make comment clearer to read --- idigbio_ingestion/mediaing/fetcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idigbio_ingestion/mediaing/fetcher.py b/idigbio_ingestion/mediaing/fetcher.py index 05312cc1..b0677ee5 100644 --- a/idigbio_ingestion/mediaing/fetcher.py +++ b/idigbio_ingestion/mediaing/fetcher.py @@ -227,7 +227,7 @@ class FetchItem(object): #: Number of concurrent connections allowed FETCHER_COUNT = 2 - #: Can http connections be reuesed? + #: Allow or disallow http connections be re-used REUSE_CONN = True #: class (static) variable for the session to use From ce129c8941635ec958fa614180d568479357cb91 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 11 Apr 2021 13:15:28 -0400 Subject: [PATCH 098/175] remove TODO we do not really need to do --- idigbio_ingestion/mediaing/fetcher.py | 1 - 1 file changed, 1 deletion(-) diff --git a/idigbio_ingestion/mediaing/fetcher.py b/idigbio_ingestion/mediaing/fetcher.py index b0677ee5..133e951c 100644 --- a/idigbio_ingestion/mediaing/fetcher.py +++ b/idigbio_ingestion/mediaing/fetcher.py @@ -34,7 +34,6 @@ LAST_CHECK_INTERVAL = '1 month' -# TODO: Add idb __version__ to this string? USER_AGENT = 'iDigBio Media Ingestor (idigbio@acis.ufl.edu https://www.idigbio.org/wiki/index.php/Media_Ingestor)' logger.info("Started fetcher with User Agent string: '{0}'".format(USER_AGENT)) From a6d1c6737a2dcede4d23ebc98e5ff0eb7b67a2bc Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 11 Apr 2021 14:25:06 -0400 Subject: [PATCH 099/175] use raw string for regex with backslash escapes --- idigbio_ingestion/mediaing/fetcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idigbio_ingestion/mediaing/fetcher.py b/idigbio_ingestion/mediaing/fetcher.py index 133e951c..7a7ab75e 100644 --- a/idigbio_ingestion/mediaing/fetcher.py +++ b/idigbio_ingestion/mediaing/fetcher.py @@ -37,7 +37,7 @@ USER_AGENT = 'iDigBio Media Ingestor (idigbio@acis.ufl.edu https://www.idigbio.org/wiki/index.php/Media_Ingestor)' logger.info("Started fetcher with User Agent string: '{0}'".format(USER_AGENT)) -PREFIX_RE = re.compile('^https?://[^/]*[/?]') +PREFIX_RE = re.compile(r'^https?://[^/]*[/?]') def once(prefix=None, ignores=IGNORE_PREFIXES): From 392255af52f18a5b53694e72b1f69dd6fa76e647 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 11 Apr 2021 14:26:19 -0400 Subject: [PATCH 100/175] add more virtualenv dirnames to gitignore --- .gitignore | 1 + idb/data_tables/build_taxon_index.py | 2 +- idb/data_tables/taxon.py | 3 +-- idb/helpers/__init__.py | 2 +- idb/helpers/conversions.py | 2 +- idb/indexing/index_helper.py | 2 +- idigbio_ingestion/lib/delimited.py | 2 +- idigbio_ingestion/lib/dwca.py | 2 +- idigbio_ingestion/lib/fileproxy.py | 2 +- idigbio_ingestion/verify_ceph_objects.py | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 4bae4e21..29f5cb1b 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,7 @@ venv/ ENV/ .venv/ .virtualenv/ +.venv*/ # Spyder project settings .spyderproject diff --git a/idb/data_tables/build_taxon_index.py b/idb/data_tables/build_taxon_index.py index f65ad0d4..bb6a0ac7 100644 --- a/idb/data_tables/build_taxon_index.py +++ b/idb/data_tables/build_taxon_index.py @@ -92,7 +92,7 @@ def get_data(path_to_checklist): def main(): # for r in get_data(os.path.expanduser("~/Downloads/backbone-current-sorted.zip")): # pass - for r in bulk_index(get_data(os.path.expanduser("~/Downloads/backbone-current-sorted.zip"))): + for _ in bulk_index(get_data(os.path.expanduser("~/Downloads/backbone-current-sorted.zip"))): pass diff --git a/idb/data_tables/taxon.py b/idb/data_tables/taxon.py index 14386eb7..f46ce259 100644 --- a/idb/data_tables/taxon.py +++ b/idb/data_tables/taxon.py @@ -122,7 +122,6 @@ def get_sources(): def main(): count = 0 - for dr in get_data(os.path.expanduser("~/Downloads/backbone-current-sorted.zip")): + for _ in get_data(os.path.expanduser("~/Downloads/backbone-current-sorted.zip")): count += 1 - # print dr print (count) diff --git a/idb/helpers/__init__.py b/idb/helpers/__init__.py index 76ace50c..4183d6b7 100644 --- a/idb/helpers/__init__.py +++ b/idb/helpers/__init__.py @@ -16,7 +16,7 @@ def first(iterable, key=None, default=None): def ilen(iterable): "Traverse (possibly consuming) an iterable to count its length" count = 0 - for e in iterable: + for _ in iterable: count += 1 return count diff --git a/idb/helpers/conversions.py b/idb/helpers/conversions.py index 677ac320..29bd7aec 100644 --- a/idb/helpers/conversions.py +++ b/idb/helpers/conversions.py @@ -801,7 +801,7 @@ def collect_common_names(t, d): if dcn is not None: cns.append(dcn) - for vn in vns: + for _ in vns: if d.get("dwc:vernacularname") is not None: cns.append(d.get("dwc:vernacularname")) diff --git a/idb/indexing/index_helper.py b/idb/indexing/index_helper.py index 74fe64ce..a3ed88cf 100644 --- a/idb/indexing/index_helper.py +++ b/idb/indexing/index_helper.py @@ -36,7 +36,7 @@ def index_record(ei, rc, typ, r, do_index=True): i["records"] = sibs.get('record', []) return (typ, i) else: - d, ck = rc.correct_record(r["data"]) + d, _ = rc.correct_record(r["data"]) d.update({ "idigbio:uuid": r["uuid"], diff --git a/idigbio_ingestion/lib/delimited.py b/idigbio_ingestion/lib/delimited.py index cbb64d47..81365232 100644 --- a/idigbio_ingestion/lib/delimited.py +++ b/idigbio_ingestion/lib/delimited.py @@ -196,6 +196,6 @@ def readlines(self, sizehint=None): Returns all lines in the file. Cheats off readline. """ lines = [] - for line in self: + for _ in self: lines.append(self.readline()) return lines diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index aa4527c5..3e189e07 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -206,7 +206,7 @@ def __init__(self,filedict,fh,logname=None): # the purpose of this is to skip any header rows while ignoreheader > 0: - next_line = self.readline() + _ = self.readline() ignoreheader -= 1 def __iter__(self): diff --git a/idigbio_ingestion/lib/fileproxy.py b/idigbio_ingestion/lib/fileproxy.py index 99dae8f6..e039129d 100644 --- a/idigbio_ingestion/lib/fileproxy.py +++ b/idigbio_ingestion/lib/fileproxy.py @@ -111,7 +111,7 @@ def main(): with FileProxy(open("test.csv", "rb")) as inf: try: cr = csv.reader(inf) - for l in cr: + for _ in cr: pass inf.snap() except: diff --git a/idigbio_ingestion/verify_ceph_objects.py b/idigbio_ingestion/verify_ceph_objects.py index bf5d532c..9a54ce0c 100644 --- a/idigbio_ingestion/verify_ceph_objects.py +++ b/idigbio_ingestion/verify_ceph_objects.py @@ -99,7 +99,7 @@ def main(poolsize=50, limit=5000): objects = get_all_objects(limit) check_results = p.imap_unordered(check_object, objects) check_results = log_counts(check_results) - for cr in check_results: + for _ in check_results: pass except KeyboardInterrupt: p.kill() From a3866decca3a4b953482c5bc489d8711a81f9931 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 11 Apr 2021 14:30:51 -0400 Subject: [PATCH 101/175] fix W0612 unused-variable linting For variables that are never used, especially in iteration, use the special _ variable name. or are simply used for iteration an --- idb/data_tables/build_taxon_index.py | 2 +- idb/data_tables/taxon.py | 3 ++- idb/helpers/__init__.py | 2 +- idb/helpers/conversions.py | 2 +- idb/indexing/index_helper.py | 2 +- idigbio_ingestion/lib/delimited.py | 2 +- idigbio_ingestion/lib/dwca.py | 2 +- idigbio_ingestion/lib/fileproxy.py | 2 +- idigbio_ingestion/verify_ceph_objects.py | 2 +- 9 files changed, 10 insertions(+), 9 deletions(-) diff --git a/idb/data_tables/build_taxon_index.py b/idb/data_tables/build_taxon_index.py index bb6a0ac7..f65ad0d4 100644 --- a/idb/data_tables/build_taxon_index.py +++ b/idb/data_tables/build_taxon_index.py @@ -92,7 +92,7 @@ def get_data(path_to_checklist): def main(): # for r in get_data(os.path.expanduser("~/Downloads/backbone-current-sorted.zip")): # pass - for _ in bulk_index(get_data(os.path.expanduser("~/Downloads/backbone-current-sorted.zip"))): + for r in bulk_index(get_data(os.path.expanduser("~/Downloads/backbone-current-sorted.zip"))): pass diff --git a/idb/data_tables/taxon.py b/idb/data_tables/taxon.py index f46ce259..14386eb7 100644 --- a/idb/data_tables/taxon.py +++ b/idb/data_tables/taxon.py @@ -122,6 +122,7 @@ def get_sources(): def main(): count = 0 - for _ in get_data(os.path.expanduser("~/Downloads/backbone-current-sorted.zip")): + for dr in get_data(os.path.expanduser("~/Downloads/backbone-current-sorted.zip")): count += 1 + # print dr print (count) diff --git a/idb/helpers/__init__.py b/idb/helpers/__init__.py index 4183d6b7..76ace50c 100644 --- a/idb/helpers/__init__.py +++ b/idb/helpers/__init__.py @@ -16,7 +16,7 @@ def first(iterable, key=None, default=None): def ilen(iterable): "Traverse (possibly consuming) an iterable to count its length" count = 0 - for _ in iterable: + for e in iterable: count += 1 return count diff --git a/idb/helpers/conversions.py b/idb/helpers/conversions.py index 29bd7aec..677ac320 100644 --- a/idb/helpers/conversions.py +++ b/idb/helpers/conversions.py @@ -801,7 +801,7 @@ def collect_common_names(t, d): if dcn is not None: cns.append(dcn) - for _ in vns: + for vn in vns: if d.get("dwc:vernacularname") is not None: cns.append(d.get("dwc:vernacularname")) diff --git a/idb/indexing/index_helper.py b/idb/indexing/index_helper.py index a3ed88cf..74fe64ce 100644 --- a/idb/indexing/index_helper.py +++ b/idb/indexing/index_helper.py @@ -36,7 +36,7 @@ def index_record(ei, rc, typ, r, do_index=True): i["records"] = sibs.get('record', []) return (typ, i) else: - d, _ = rc.correct_record(r["data"]) + d, ck = rc.correct_record(r["data"]) d.update({ "idigbio:uuid": r["uuid"], diff --git a/idigbio_ingestion/lib/delimited.py b/idigbio_ingestion/lib/delimited.py index 81365232..cbb64d47 100644 --- a/idigbio_ingestion/lib/delimited.py +++ b/idigbio_ingestion/lib/delimited.py @@ -196,6 +196,6 @@ def readlines(self, sizehint=None): Returns all lines in the file. Cheats off readline. """ lines = [] - for _ in self: + for line in self: lines.append(self.readline()) return lines diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index 3e189e07..aa4527c5 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -206,7 +206,7 @@ def __init__(self,filedict,fh,logname=None): # the purpose of this is to skip any header rows while ignoreheader > 0: - _ = self.readline() + next_line = self.readline() ignoreheader -= 1 def __iter__(self): diff --git a/idigbio_ingestion/lib/fileproxy.py b/idigbio_ingestion/lib/fileproxy.py index e039129d..99dae8f6 100644 --- a/idigbio_ingestion/lib/fileproxy.py +++ b/idigbio_ingestion/lib/fileproxy.py @@ -111,7 +111,7 @@ def main(): with FileProxy(open("test.csv", "rb")) as inf: try: cr = csv.reader(inf) - for _ in cr: + for l in cr: pass inf.snap() except: diff --git a/idigbio_ingestion/verify_ceph_objects.py b/idigbio_ingestion/verify_ceph_objects.py index 9a54ce0c..bf5d532c 100644 --- a/idigbio_ingestion/verify_ceph_objects.py +++ b/idigbio_ingestion/verify_ceph_objects.py @@ -99,7 +99,7 @@ def main(poolsize=50, limit=5000): objects = get_all_objects(limit) check_results = p.imap_unordered(check_object, objects) check_results = log_counts(check_results) - for _ in check_results: + for cr in check_results: pass except KeyboardInterrupt: p.kill() From 11257355158320d24d15bc1550c7c4bae8b76e19 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 11 Apr 2021 14:34:11 -0400 Subject: [PATCH 102/175] Revert "fix W0612 unused-variable linting" This reverts commit a3866decca3a4b953482c5bc489d8711a81f9931. Too many reverts, we actually want everything here. --- idb/data_tables/build_taxon_index.py | 2 +- idb/data_tables/taxon.py | 3 +-- idb/helpers/__init__.py | 2 +- idb/helpers/conversions.py | 2 +- idb/indexing/index_helper.py | 2 +- idigbio_ingestion/lib/delimited.py | 2 +- idigbio_ingestion/lib/dwca.py | 2 +- idigbio_ingestion/lib/fileproxy.py | 2 +- idigbio_ingestion/verify_ceph_objects.py | 2 +- 9 files changed, 9 insertions(+), 10 deletions(-) diff --git a/idb/data_tables/build_taxon_index.py b/idb/data_tables/build_taxon_index.py index f65ad0d4..bb6a0ac7 100644 --- a/idb/data_tables/build_taxon_index.py +++ b/idb/data_tables/build_taxon_index.py @@ -92,7 +92,7 @@ def get_data(path_to_checklist): def main(): # for r in get_data(os.path.expanduser("~/Downloads/backbone-current-sorted.zip")): # pass - for r in bulk_index(get_data(os.path.expanduser("~/Downloads/backbone-current-sorted.zip"))): + for _ in bulk_index(get_data(os.path.expanduser("~/Downloads/backbone-current-sorted.zip"))): pass diff --git a/idb/data_tables/taxon.py b/idb/data_tables/taxon.py index 14386eb7..f46ce259 100644 --- a/idb/data_tables/taxon.py +++ b/idb/data_tables/taxon.py @@ -122,7 +122,6 @@ def get_sources(): def main(): count = 0 - for dr in get_data(os.path.expanduser("~/Downloads/backbone-current-sorted.zip")): + for _ in get_data(os.path.expanduser("~/Downloads/backbone-current-sorted.zip")): count += 1 - # print dr print (count) diff --git a/idb/helpers/__init__.py b/idb/helpers/__init__.py index 76ace50c..4183d6b7 100644 --- a/idb/helpers/__init__.py +++ b/idb/helpers/__init__.py @@ -16,7 +16,7 @@ def first(iterable, key=None, default=None): def ilen(iterable): "Traverse (possibly consuming) an iterable to count its length" count = 0 - for e in iterable: + for _ in iterable: count += 1 return count diff --git a/idb/helpers/conversions.py b/idb/helpers/conversions.py index 677ac320..29bd7aec 100644 --- a/idb/helpers/conversions.py +++ b/idb/helpers/conversions.py @@ -801,7 +801,7 @@ def collect_common_names(t, d): if dcn is not None: cns.append(dcn) - for vn in vns: + for _ in vns: if d.get("dwc:vernacularname") is not None: cns.append(d.get("dwc:vernacularname")) diff --git a/idb/indexing/index_helper.py b/idb/indexing/index_helper.py index 74fe64ce..a3ed88cf 100644 --- a/idb/indexing/index_helper.py +++ b/idb/indexing/index_helper.py @@ -36,7 +36,7 @@ def index_record(ei, rc, typ, r, do_index=True): i["records"] = sibs.get('record', []) return (typ, i) else: - d, ck = rc.correct_record(r["data"]) + d, _ = rc.correct_record(r["data"]) d.update({ "idigbio:uuid": r["uuid"], diff --git a/idigbio_ingestion/lib/delimited.py b/idigbio_ingestion/lib/delimited.py index cbb64d47..81365232 100644 --- a/idigbio_ingestion/lib/delimited.py +++ b/idigbio_ingestion/lib/delimited.py @@ -196,6 +196,6 @@ def readlines(self, sizehint=None): Returns all lines in the file. Cheats off readline. """ lines = [] - for line in self: + for _ in self: lines.append(self.readline()) return lines diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index aa4527c5..3e189e07 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -206,7 +206,7 @@ def __init__(self,filedict,fh,logname=None): # the purpose of this is to skip any header rows while ignoreheader > 0: - next_line = self.readline() + _ = self.readline() ignoreheader -= 1 def __iter__(self): diff --git a/idigbio_ingestion/lib/fileproxy.py b/idigbio_ingestion/lib/fileproxy.py index 99dae8f6..e039129d 100644 --- a/idigbio_ingestion/lib/fileproxy.py +++ b/idigbio_ingestion/lib/fileproxy.py @@ -111,7 +111,7 @@ def main(): with FileProxy(open("test.csv", "rb")) as inf: try: cr = csv.reader(inf) - for l in cr: + for _ in cr: pass inf.snap() except: diff --git a/idigbio_ingestion/verify_ceph_objects.py b/idigbio_ingestion/verify_ceph_objects.py index bf5d532c..9a54ce0c 100644 --- a/idigbio_ingestion/verify_ceph_objects.py +++ b/idigbio_ingestion/verify_ceph_objects.py @@ -99,7 +99,7 @@ def main(poolsize=50, limit=5000): objects = get_all_objects(limit) check_results = p.imap_unordered(check_object, objects) check_results = log_counts(check_results) - for cr in check_results: + for _ in check_results: pass except KeyboardInterrupt: p.kill() From f405a848de2b6c979f3591d74e0e327c821bc8e2 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 12 Apr 2021 08:25:08 -0400 Subject: [PATCH 103/175] add four more etags to derivatives blacklist (#165) We don't need to see these failing in the log over and over. ``` Apr 11 08:16:22 c18node4 idigbio-ingestion[12220]: failed downloading with 404 Not Found 9ab4bb59a0f29df356fc2329177ce3a2 Apr 11 08:16:22 c18node4 idigbio-ingestion[12220]: failed downloading with 404 Not Found 8d170c55d606809d2f56da323271a47e Apr 11 08:16:22 c18node4 idigbio-ingestion[12220]: failed downloading with 404 Not Found 8818e0b4ba8cc7508f63ea675eef971f Apr 11 08:16:22 c18node4 idigbio-ingestion[12188]: /usr/local/lib/python2.7/dist-packages/PIL/TiffImagePlugin.py:725: UserWarning: Possibly corrupt EXIF data. Expecting to read 14 bytes but only got 8. Skipping tag 42036 ``` --- idb/blacklists/derivatives.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/idb/blacklists/derivatives.py b/idb/blacklists/derivatives.py index 09ed8954..fa61208d 100644 --- a/idb/blacklists/derivatives.py +++ b/idb/blacklists/derivatives.py @@ -640,6 +640,7 @@ '872795e5abf5b3adc08c33d4bd2450d7', '877f285468ce60b1b4f9aa3bac7c5f17', '87e1b58ae4a4959848283a0247866407', + '8818e0b4ba8cc7508f63ea675eef971f', '8845c2783972614f263fda4050938857', '888139cef0fb89bf3f49258e5ef68f4c', '88c7f35d0e01ec843a5fe6d82568408d', @@ -667,6 +668,7 @@ '8ce6bd7e475f0272889e25e8660ebb66', '8cf0e04fee96a4b641c0bbecae3036f7', '8d119f3c73318c23fb9a13c74f9278a3', + '8d170c55d606809d2f56da323271a47e', '8d26165011a96624847534d6ff1e5d35', '8d4d0ca7699a9f02311aefc59abf8560', '8d650252168d646e8950d9f94d25c144', @@ -722,6 +724,7 @@ '9a50f8706406632573eb5ec519371c47', '9a5b78f794d75355d59ee7719127ddf3', '9a791f306d3c493fa28b360d036a6965', + '9ab4bb59a0f29df356fc2329177ce3a2', '9adc0a0e90fc7428bd7d78a081543d90', '9b0153790035845f127637309a72fdfc', '9b0f1f8d0c17aee0b49df0382958e471', @@ -957,6 +960,7 @@ 'ce4ea3fcebd1f30367445122d3bd31b6', 'ce95ecbbc8b78cbef52ab86ca4888244', 'cef2fd47dde23bfa38477243250f1c5c', + 'cf369361edab448ce4570cf9d8a1cdd3', 'cf7a0f2620142e307e032e3daab72ff7', 'd06423a5a0e7c00a97f71c5645836ba6', 'd0bd37efeb735c0ca77ceb645235beb4', From 0184486725a40fad4a641c0e96ff3f2880fe90b4 Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Mon, 19 Apr 2021 21:31:34 +0000 Subject: [PATCH 104/175] Minor spelling correction. --- idb/helpers/logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idb/helpers/logging.py b/idb/helpers/logging.py index 02ad19dd..55fdd538 100644 --- a/idb/helpers/logging.py +++ b/idb/helpers/logging.py @@ -27,7 +27,7 @@ LOGBOOK_FORMAT_STRING = u'{record.time:%Y-%m-%d %H:%M:%S.%f} {record.level_name:<5.5} ' \ u'{record.channel}\u10fb {record.message}' -#: Libaries used whos logs should be at a higher level. +#: Libaries used whose logs should be at a higher level. LIBRARY_LOGGERS = ('boto', 'requests', 'urllib3', 'elasticsearch', 'shapely', 'gipc') From c2b4b43df1ac572b969020e2751235a9a934ad01 Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Mon, 19 Apr 2021 21:33:21 +0000 Subject: [PATCH 105/175] Address issue #123, now development mode provides information when starting. --- idb/data_api/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/idb/data_api/__init__.py b/idb/data_api/__init__.py index f2795781..1848303f 100644 --- a/idb/data_api/__init__.py +++ b/idb/data_api/__init__.py @@ -74,8 +74,9 @@ def run_server(info, host, port, reload, debugger, eager_loading, debug, wsgi): # import path because the app was loaded through a callback then # we won't print anything. if info.app_import_path is not None: - print("Werkzeug server @ http://{0}:{1}/ ENV={2}".format(host, port, config.ENV), - file=sys.stderr) + # logger is not started yet. + startup_msg = "Werkzeug server @ http://{0}:{1}/ ENV={2}".format(host, port, config.ENV) + print(startup_msg, file=sys.stderr) if info.debug is not None: print(' * Forcing debug %s' % (info.debug and 'on' or 'off')) @@ -91,7 +92,11 @@ def run_server(info, host, port, reload, debugger, eager_loading, debug, wsgi): logger = idblogger.getChild('api') from werkzeug.middleware.proxy_fix import ProxyFix - logger.info("gevent server @ http://%s:%s/ ENV=%s", host, port, config.ENV) + startup_msg = "gevent server @ http://{0}:{1}/ ENV={2}".format(host, port, config.ENV) + if logger.level < 1: # more severe than logging.INFO, so we need to print it + print(startup_msg, file=sys.stderr) + else: + logger.info(startup_msg) app = info.load_app() app = WSGILogger(app, [], ApacheFormatter()) app.logger = logger.getChild('r') From 9219d2f399bfba4ac106ddeb638facdfcaf79cd0 Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Thu, 6 May 2021 19:14:32 +0000 Subject: [PATCH 106/175] Adding comment to conversions code to show where flag is set. --- idb/helpers/conversions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/idb/helpers/conversions.py b/idb/helpers/conversions.py index 29bd7aec..90c83c61 100644 --- a/idb/helpers/conversions.py +++ b/idb/helpers/conversions.py @@ -432,7 +432,8 @@ def geoGrabber(t, d): latexp = getExponent(lat_val) lonexp = getExponent(lon_val) - + # geopoint_pre_flip flag set here + # if latitude and longitude fall in ranges below, swap them if ( (-180 <= lat < -90 or 90 < lat <= 180) and (-90 <= lon <= 90) From 3e74562d4329f80efe7843fc6658e9370839959c Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Thu, 6 May 2021 19:16:42 +0000 Subject: [PATCH 107/175] Adding comment to conversions code to show where flag is set. --- idb/helpers/conversions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/idb/helpers/conversions.py b/idb/helpers/conversions.py index 90c83c61..c9d32668 100644 --- a/idb/helpers/conversions.py +++ b/idb/helpers/conversions.py @@ -440,7 +440,9 @@ def geoGrabber(t, d): ): lat, lon = lon, lat r["flag_geopoint_pre_flip"] = True - + # geopoint_bounds flag set here + # latitude must be in the range -90 to 90 + # -90 corresponds to South Pole, 90 North Pole if not (-90 <= lat <= 90): r["geopoint"] = None r["flag_geopoint_bounds"] = True From 7c0ccc710001b5b13dfd6e46ed93663cc6730918 Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Thu, 6 May 2021 19:18:29 +0000 Subject: [PATCH 108/175] Adding comment to conversions code to show where flag is set. --- idb/helpers/conversions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/idb/helpers/conversions.py b/idb/helpers/conversions.py index c9d32668..eb3a2fdc 100644 --- a/idb/helpers/conversions.py +++ b/idb/helpers/conversions.py @@ -447,7 +447,8 @@ def geoGrabber(t, d): r["geopoint"] = None r["flag_geopoint_bounds"] = True return r - + # geopoint_bounds flag set here also + # longitude must in the range -180 to 180 if not (-180 <= lon <= 180): r["geopoint"] = None r["flag_geopoint_bounds"] = True From c3da5d65c17825e92bacedbfead649b3722990d7 Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Thu, 6 May 2021 19:36:09 +0000 Subject: [PATCH 109/175] Add docstring for idb.helpers.conversion.checkBounds --- idb/helpers/conversions.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/idb/helpers/conversions.py b/idb/helpers/conversions.py index eb3a2fdc..63df4607 100644 --- a/idb/helpers/conversions.py +++ b/idb/helpers/conversions.py @@ -223,6 +223,15 @@ def checkBounds(x): + """ + Checks whether a supplied date is within the agreed timeframe. + iDigBio only accepts data from 1700-01-02 UTC to now (when ingestion runs.) + x: datetime.datetime + Returns + ------- + bool + Indicates whether the date is allowed. + """ lowerBound = datetime.datetime(1700, 1, 2, tzinfo=pytz.utc) upperBound = datetime.datetime.now(pytz.utc) if isinstance(x, datetime.datetime): From 710c98e014fa6fb5aa78485e447bf20bc6784847 Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Thu, 6 May 2021 20:00:49 +0000 Subject: [PATCH 110/175] Add unit test for idb.helpers.conversions.checkBounds. --- tests/idb/test_helpers_conversions.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/idb/test_helpers_conversions.py b/tests/idb/test_helpers_conversions.py index 12995a22..73024e43 100644 --- a/tests/idb/test_helpers_conversions.py +++ b/tests/idb/test_helpers_conversions.py @@ -13,6 +13,15 @@ def test_get_exponent(self): for i,exp in enumerate(exponents): self.assertEqual(i, conversions.getExponent(exp)) +class TestCheckBounds(unittest.TestCase): + def test_check_bounds(self): + bad_past_date = datetime.datetime(1500, 1, 1, tzinfo=pytz.utc) + bad_future_date = datetime.datetime.now(pytz.utc) + datetime.timedelta(100) + good_past_date = datetime.datetime(1700, 12, 12, tzinfo=pytz.utc) + self.assertEqual(True, conversions.checkBounds(bad_past_date)) + self.assertEqual(True, conversions.checkBounds(bad_future_date)) + self.assertEqual(False, conversions.checkBounds(good_past_date)) + class TestSetFlags(unittest.TestCase): def test_set_flags(self): From 94d82117f4875d0decadb7591d8aa88357366ace Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Thu, 17 Jun 2021 20:50:56 -0400 Subject: [PATCH 111/175] fix use of dict.keys() which has different behavior in py3 --- idigbio_ingestion/db_check.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/idigbio_ingestion/db_check.py b/idigbio_ingestion/db_check.py index 88198ca7..aef86313 100644 --- a/idigbio_ingestion/db_check.py +++ b/idigbio_ingestion/db_check.py @@ -301,7 +301,7 @@ def process_subfile(rf, rsid, rs_uuid_etag, rs_id_uuid, ingest=False, db=None): else: if config.IDB_EXTRA_SERIOUS_DEBUG == 'yes': rlogger.debug("Setting sibling for '{0}'".format(u)) - db.set_record(u, typ[:-1], rsid, r, ids_to_add.keys(), siblings) + db.set_record(u, typ[:-1], rsid, r, [*ids_to_add.keys()], siblings) ingestions += 1 elif ingest and deleted: db.undelete_item(u) @@ -392,8 +392,8 @@ def process_subfile(rf, rsid, rs_uuid_etag, rs_id_uuid, ingest=False, db=None): seen_ids.update(ids_to_add) seen_uuids.update(uuids_to_add) - eu_set = existing_etags.keys() - nu_set = seen_uuids.keys() + eu_set = set(existing_etags.keys()) + nu_set = set(seen_uuids.keys()) deletes = len(eu_set - nu_set) From 3ed14f6656ca8de64463c8112efbb01e5db70e7c Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Thu, 24 Jun 2021 22:56:44 -0400 Subject: [PATCH 112/175] Fixing W1651 deprecated-itertools-function: itertools.imap is replaced by map in Python 3. --- idb/clibase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idb/clibase.py b/idb/clibase.py index 14897137..bc219c97 100644 --- a/idb/clibase.py +++ b/idb/clibase.py @@ -104,7 +104,7 @@ def wrapper(*args, **kwargs): else: fnname = getattr(fn, 'func_name', fn) argsreprs = itertools.chain( - itertools.imap(repr, args), + map(repr, args), itertools.starmap(u'{0!s}={1!r}'.format, kwargs.items())) logfn('DRY_RUN: {0}({1})', fnname, u', '.join(argsreprs)) return retval From 6907eec72eaa12c2c87a77216de8241c2e0dad0b Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Thu, 24 Jun 2021 23:05:39 -0400 Subject: [PATCH 113/175] Addressing W1620: Calling a dict.iter*() method (dict-iter-method), replacing dict.iteritems() with iter(dict.items()) --- idb/data_tables/locality.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idb/data_tables/locality.py b/idb/data_tables/locality.py index 667e49eb..619f594e 100644 --- a/idb/data_tables/locality.py +++ b/idb/data_tables/locality.py @@ -24,7 +24,7 @@ def get_data(): to_insert = [] for k in vds: - for v, o in vds[k].iteritems(): + for v, o in iter(vds[k].items()): if "dwc:country" in o and o["dwc:country"] in vds["dwc:country"]: o.update(vds["dwc:country"][o["dwc:country"]]) to_insert.append((dict([[k, v]]), o, "data_dictionaries_1", True)) From 1240fbfcc7a8c8902e235799447ea4028a83130d Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Thu, 24 Jun 2021 23:09:24 -0400 Subject: [PATCH 114/175] Addressing W1633: round built-in referenced (round-builtin). The round built-in is in both Python 2 and 3, but the underlying rounding algorithm has changed, hence the warning. --- idb/data_tables/build_taxon_kv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idb/data_tables/build_taxon_kv.py b/idb/data_tables/build_taxon_kv.py index 5a6fbf1b..2d0d119f 100644 --- a/idb/data_tables/build_taxon_kv.py +++ b/idb/data_tables/build_taxon_kv.py @@ -62,7 +62,7 @@ def run_query(q, cache_string, log_score=True): #search_cache[cache_string] = None if best_response is not None: if log_score: - score_stats[round(best_response["_score"], 1)] += 1 + score_stats[round(best_response["_score"], 1)] += 1 # noqa # Reject low quality matches # Disable fixed cutoff here, moving to loader script and using first quartile From 2acb2d522dac9aa39b4a77d48e248c94ff5cbcf5 Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Thu, 24 Jun 2021 23:16:57 -0400 Subject: [PATCH 115/175] Change ignore statement since we're using pylint, not flake or other QA tool. --- idb/data_tables/build_taxon_kv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idb/data_tables/build_taxon_kv.py b/idb/data_tables/build_taxon_kv.py index 2d0d119f..0690c2a1 100644 --- a/idb/data_tables/build_taxon_kv.py +++ b/idb/data_tables/build_taxon_kv.py @@ -62,7 +62,7 @@ def run_query(q, cache_string, log_score=True): #search_cache[cache_string] = None if best_response is not None: if log_score: - score_stats[round(best_response["_score"], 1)] += 1 # noqa + score_stats[round(best_response["_score"], 1)] += 1 # pylint: disable=round-builtin # Reject low quality matches # Disable fixed cutoff here, moving to loader script and using first quartile From f945f7c21332b465db95a6843c26aeaed811daa2 Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Thu, 24 Jun 2021 23:22:10 -0400 Subject: [PATCH 116/175] Revert "Change ignore statement since we're using pylint, not flake or other QA tool." This reverts commit 2acb2d522dac9aa39b4a77d48e248c94ff5cbcf5. --- idb/data_tables/build_taxon_kv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idb/data_tables/build_taxon_kv.py b/idb/data_tables/build_taxon_kv.py index 0690c2a1..2d0d119f 100644 --- a/idb/data_tables/build_taxon_kv.py +++ b/idb/data_tables/build_taxon_kv.py @@ -62,7 +62,7 @@ def run_query(q, cache_string, log_score=True): #search_cache[cache_string] = None if best_response is not None: if log_score: - score_stats[round(best_response["_score"], 1)] += 1 # pylint: disable=round-builtin + score_stats[round(best_response["_score"], 1)] += 1 # noqa # Reject low quality matches # Disable fixed cutoff here, moving to loader script and using first quartile From 4053998f94dcbe1eac913a94316491f0da2cdba3 Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Thu, 24 Jun 2021 23:22:21 -0400 Subject: [PATCH 117/175] Revert "Addressing W1633: round built-in referenced (round-builtin). The round built-in is in both Python 2 and 3, but the underlying rounding algorithm has changed, hence the warning." This reverts commit 1240fbfcc7a8c8902e235799447ea4028a83130d. --- idb/data_tables/build_taxon_kv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idb/data_tables/build_taxon_kv.py b/idb/data_tables/build_taxon_kv.py index 2d0d119f..5a6fbf1b 100644 --- a/idb/data_tables/build_taxon_kv.py +++ b/idb/data_tables/build_taxon_kv.py @@ -62,7 +62,7 @@ def run_query(q, cache_string, log_score=True): #search_cache[cache_string] = None if best_response is not None: if log_score: - score_stats[round(best_response["_score"], 1)] += 1 # noqa + score_stats[round(best_response["_score"], 1)] += 1 # Reject low quality matches # Disable fixed cutoff here, moving to loader script and using first quartile From 4a4f8e9efaa03193e9f4718096ac508dba5c42a3 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 4 Aug 2021 18:28:16 -0400 Subject: [PATCH 118/175] add note about ssh port forwarding to access Elasticsearch --- tests/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/README.md b/tests/README.md index 0de3276e..1d0e1fc0 100644 --- a/tests/README.md +++ b/tests/README.md @@ -108,6 +108,14 @@ Some idb-backend tests depend on external resources, such as a local test postgr * Tests that depend on Elasticsearch will FAIL if the Elasticsearch cluster cannot be reached (fail very slowly in fact), or if there is some other failure. +Due to network access control it might be necessary to use ssh port forwarding. + +``` +# replace ELASTICSEARCH_CLUSTER_NODE_IP, USER, SSH_HOST with real values. +$ ssh -nNT -L 9200:ELASTICSEARCH_CLUSTER_NODE_IP:9200 USER@SSH_HOST +``` + + The local postgresql 9.5 DB is named `test_idigbio` with user/pass `test` / `test`. Note: The data in the db with that name will be destroyed during testing. From 029b585c05e5d27bb6d7e59bb2e4310d070a3374 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 16 Aug 2021 20:31:09 -0400 Subject: [PATCH 119/175] update README with howto get idb binaries in path --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 9903e620..9dcf47bd 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,16 @@ no need to set the `PYTHONPATH`. In any invocation an `idigbio.json` must be present in either `$PWD`, `$HOME`, or `/etc/idigbio`. +For convenience we can create links to the virtual environment entrypoints in `/usr/local/bin/` so we do not need to activate the virtual environment explicitly or use the full path. + +``` +$ sudo ln -s /home/idigbio-ingestion/idb-backend/venv/bin/idb /usr/local/bin/idb +$ ls -la /usr/local/bin/idb +lrwxrwxrwx 1 root root 48 Aug 16 20:23 /usr/local/bin/idb -> /home/idigbio-ingestion/idb-backend/venv/bin/idb +$ sudo ln -s /home/idigbio-ingestion/idb-backend/venv/bin/idigbio-ingestion /usr/local/bin/idigbio-ingestion +$ ls -la /usr/local/bin/idigbio-ingestion +lrwxrwxrwx 1 root root 62 Aug 16 20:24 /usr/local/bin/idigbio-ingestion -> /home/idigbio-ingestion/idb-backend/venv/bin/idigbio-ingestion +``` ### Data API This serves the `api.idigbio.org` interface From c55aabfc463ca17055bfbedada1d230821af393c Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 23 Nov 2021 23:33:16 -0500 Subject: [PATCH 120/175] add note - try to run ES in docker --- tests/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 1d0e1fc0..2dd111d9 100644 --- a/tests/README.md +++ b/tests/README.md @@ -134,9 +134,11 @@ Consider running elasticsearch the same way we run postrgres... ``` $ docker pull docker.elastic.co/elasticsearch/elasticsearch:5.5.3 -$ docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:5.5.3 +$ docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e "xpack.security.enabled=false" docker.elastic.co/elasticsearch/elasticsearch:5.5.3 ``` +One challenge is the test suite expects mappings, etc. to be present. + To do: 1. Have tests connect to localhost instead of the ES cluster that exists in CONFIG. From f87db0dd9635cfe5de4ae941bad198f73b1fcf8b Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Fri, 17 Dec 2021 05:11:17 -0500 Subject: [PATCH 121/175] remove link to vertnet since we no longer have access --- scripts/data-ingestion-status-report.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/data-ingestion-status-report.py b/scripts/data-ingestion-status-report.py index 3c858628..799fcb31 100644 --- a/scripts/data-ingestion-status-report.py +++ b/scripts/data-ingestion-status-report.py @@ -106,7 +106,6 @@ (login required) https://www.idigbio.org/redmine/projects/data-mobilization/issues -https://github.com/vertnet/tasks ''' From ca9ca13764729bcf39c1453e7ffc2e460e540636 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 13 Feb 2022 16:34:15 -0500 Subject: [PATCH 122/175] add more media error statuses --- idigbio_ingestion/mediaing/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/idigbio_ingestion/mediaing/__init__.py b/idigbio_ingestion/mediaing/__init__.py index 26163f37..65ca5335 100644 --- a/idigbio_ingestion/mediaing/__init__.py +++ b/idigbio_ingestion/mediaing/__init__.py @@ -29,6 +29,7 @@ class Status(Enum): NOT_ACCEPTABLE = 406 REQUEST_TIMEOUT = 408 GONE = 410 + UNSUPPORTED_MEDIA_TYPE = 415 LEGAL_REASONS = 451 SERVER_ERROR = 500 @@ -38,6 +39,7 @@ class Status(Enum): GATEWAY_TIMEOUT = 504 HTTP_VERSION_NOT_SUPPORTED = 505 INSUFFICIENT_STORAGE = 507 + LOOP_DETECTED = 508 BANDWIDTH_LIMIT_EXCEEDED = 509 UNHANDLED_FAILURE = 1000 From 4d9d5c7655cd7d84ae446d8418da0db39a9e3a7b Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 19 Jan 2022 19:45:17 -0500 Subject: [PATCH 123/175] change redmine link in Ingestion report to new Phase 3 subsection (#202) --- scripts/data-ingestion-status-report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/data-ingestion-status-report.py b/scripts/data-ingestion-status-report.py index 799fcb31..99209a45 100644 --- a/scripts/data-ingestion-status-report.py +++ b/scripts/data-ingestion-status-report.py @@ -105,7 +105,7 @@ https://www.idigbio.org/wiki/index.php/Data_Ingestion_Report (login required) -https://www.idigbio.org/redmine/projects/data-mobilization/issues +https://redmine.idigbio.org/projects/data-mobilization-and-ingestion-including-via-symbiota-support-hub ''' From 253e9da7033af351c61af7d42cb28faa9ccb65bb Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Thu, 13 Jan 2022 20:23:27 -0500 Subject: [PATCH 124/175] reduce derivatives processing count to save memory (#201) --- idigbio_ingestion/mediaing/derivatives.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idigbio_ingestion/mediaing/derivatives.py b/idigbio_ingestion/mediaing/derivatives.py index 76143f13..f99128e0 100644 --- a/idigbio_ingestion/mediaing/derivatives.py +++ b/idigbio_ingestion/mediaing/derivatives.py @@ -26,7 +26,7 @@ # Some really large images starting to come from some data providers. Reduce # the pool size to keep from running the workflow machine out of memory. -POOLSIZE = 25 +POOLSIZE = 10 DTYPES = ('thumbnail', 'fullsize', 'webview') logger = idblogger.getChild('deriv') From c2788209600d5071fff2d9d71a02893b6b392c8a Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 10 Jan 2022 07:58:50 -0500 Subject: [PATCH 125/175] add new term namespaces (#199) --- idb/helpers/fieldnames.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/idb/helpers/fieldnames.py b/idb/helpers/fieldnames.py index c40ca749..27e8fd9f 100644 --- a/idb/helpers/fieldnames.py +++ b/idb/helpers/fieldnames.py @@ -27,7 +27,13 @@ "http://ns.adobe.com/exif/1.0/": "exif", "http://purl.org/NET/aec/NET/aec/": "aec", "http://purl.org/NET/aec/": "aec", - "http://zooarchnet.org/dwc/terms/": "zan" + "http://zooarchnet.org/dwc/terms/": "zan", + "http://rs.ala.org.au/terms/1.0/": "ala", + "http://rs.tdwg.org/abcd/terms/": "abcd", + "http://hiscom.chah.org.au/hispid/terms/": "hispid", + "http://data.ggbn.org/schemas/ggbn/terms/": "ggbn", + "http://rs.iobis.org/obis/terms/": "obis", + "http://rs.tdwg.org/chrono/terms/": "chrono" } namespaces_rev = {v:k for k, v in namespaces.items()} From 86df49bbe6872605bc33da0feb6c32dd85d3603f Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 9 Jan 2022 21:28:17 -0500 Subject: [PATCH 126/175] ALA new namespaces (#198) * Add new namespaces for fields published by ALA * Re-order final steps of db-check so the summary files are created before trying to post stats. * trigger fatal error when dotted fieldnames are detected While processing darwin core archives, we normally map dotted term references to a CURIE. "http://rs.tdwg.org/dwc/terms/occurrenceID" as specified in meta.xml becomes "dwc:dwc:occurrenceID" when we store the field in postgres. In situations where new terms are introduced by the community from previously-unknown namespaces, the code was previously defaulting to using the field-name as-provided (including the dots "."). This had downstream effects during Indexing because Elasticsearch no longer supports dots in fieldnames. The code changes here will prevent field names with dots from being created in the database and should prevent the negative downstream effect during indexing. --- idb/helpers/fieldnames.py | 11 ++++++++--- idigbio_ingestion/db_check.py | 11 ++++++----- idigbio_ingestion/lib/dwca.py | 18 +++++++++++++++++- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/idb/helpers/fieldnames.py b/idb/helpers/fieldnames.py index 27e8fd9f..11bc3c3b 100644 --- a/idb/helpers/fieldnames.py +++ b/idb/helpers/fieldnames.py @@ -4,7 +4,9 @@ from collections import defaultdict ''' -The namespaces are ... +Namespaces differentiate the sources of terms / definitions. +When we encounter a new term from a new namespace, the namespace should be +added here. ''' namespaces = { @@ -45,7 +47,9 @@ namespaces_rev["Iptc4xmpExt"] = "http://iptc.org/std/Iptc4xmpExt/2008-02-29/" ''' -types map the namespace URI to a Compact URI / CURIE +This "types" data structure holds all of the known Extensions. +The namespace URI is mapped to a Compact URI / CURIE. +All extensions used in any darwin core archive that we process must exist here. ''' types = { "http://purl.org/NET/aec/associatedTaxa": {"shortname": "aec:associatedTaxa"}, @@ -69,7 +73,8 @@ } ''' -The translate_dict is used to... +The translate_dict is used by get_canonical_name() to map "similar" field names to the canonical +name for the field. ''' translate_dict = { "ac:accessURI": ["ac:accessURI", "dwc:Multimedia"], diff --git a/idigbio_ingestion/db_check.py b/idigbio_ingestion/db_check.py index aef86313..7e2707a9 100644 --- a/idigbio_ingestion/db_check.py +++ b/idigbio_ingestion/db_check.py @@ -569,16 +569,17 @@ def metadataToSummaryJSON(rsid, metadata, writeFile=True, doStats=True): summary["duplicate_occurence_count"] = duplicate_record_count summary["dublicate_occurence_ids"] = duplicate_id_count - if doStats: - stats.index(doc_type='digest', body=summary) - if writeFile: with AtomicFile(rsid + ".summary.json", "w") as jf: json.dump(summary, jf, indent=2) with AtomicFile(rsid + ".metadata.json", "w") as jf: json.dump(metadata, jf, indent=2) - else: - return summary + jf.write(os.linesep) + + if doStats: + stats.index(doc_type='digest', body=summary) + + return summary def main(rsid, ingest=False): diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index 3e189e07..7c0e349b 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -187,11 +187,27 @@ def __init__(self,filedict,fh,logname=None): # drop any extra quote characters term = fld['#term'].replace("\"","") - # map xmp namespaces into short code form (xxx:fieldName), longest namespaces first + # Map fieldnames that contain a namespace to a CURIE form that includes the + # shortened namespace alias and the term name (namespace:term). + # e.g. dwc:basisOfRecord + # Sort by longest namespaces first + ns_found = False for ns in sorted(namespaces.keys(),key=lambda x: len(x), reverse=True): if term.startswith(ns): + ns_found = True term = term.replace(ns,namespaces[ns]+":") break + + if not ns_found: + self.logger.warning("Term '{0}' does not belong to a known namespace.".format(term)) + if "." in term: + # Due to downstream limitations of Elasticsearch, do not allow fieldnames containing dots. + # If we encounter a new field containing dots it usually means the namespace + # needs to be added to fieldnames.py. + self.logger.fatal("Term '{0}' contains a dot '.' which is not allowed in field names.".format(term)) + raise Exception("Term '{0}' contains a dot '.' which is not allowed in field names.".format(term)) + + self.logger.debug("Using term = '{0}'".format(term)) if '#index' in fld: if int(fld['#index']) not in fields: fields[int(fld['#index'])] = term From 139a1f55daf143182c8543b608a0abfd9390052a Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 15 Nov 2021 22:08:57 -0500 Subject: [PATCH 127/175] support https symbiota terms urls in meta.xml (#196) --- idb/helpers/fieldnames.py | 1 + 1 file changed, 1 insertion(+) diff --git a/idb/helpers/fieldnames.py b/idb/helpers/fieldnames.py index 11bc3c3b..a93857de 100644 --- a/idb/helpers/fieldnames.py +++ b/idb/helpers/fieldnames.py @@ -20,6 +20,7 @@ "http://iptc.org/std/Iptc4xmpExt/2008-02-29/": "Iptc4xmpExt", "http://portal.idigbio.org/terms/": "idigbio", "http://symbiota.org/terms/": "symbiota", + "https://symbiota.org/terms/": "symbiota", "http://portal.idigbio.org/terms/inhs/": "inhs", "http://www.w3.org/2003/01/geo/wgs84_pos#": "wgs84_pos", "http://rs.gbif.org/terms/1.0/": "gbif", From e86a6f9f42ed9ccb74189a456e08f77c6bcccc00 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sat, 13 Nov 2021 13:00:59 -0500 Subject: [PATCH 128/175] log errors during bulk indexing (#195) --- idb/indexing/index_from_postgres.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/idb/indexing/index_from_postgres.py b/idb/indexing/index_from_postgres.py index eb75b6fb..e1388a5b 100644 --- a/idb/indexing/index_from_postgres.py +++ b/idb/indexing/index_from_postgres.py @@ -309,9 +309,15 @@ def consume(ei, rc, iter_func, no_index=False): pass else: for ok, item in ei.bulk_index(index_record_tuples): - pass - gc.collect() - + # Is there a way to try/except the iterator to prevent Exceptions from being fatal? + # Let's try! + # + # pass + if not ok: + logger.warning('Failed during bulk index index: {0} '.format(item)) + # We should never need to call gc manually. Can we drop this? Especially + # since we no longer ever use continuous mode. + gc.collect() def continuous_incremental(ei, rc, no_index=False): while True: From fb1bcaadb7cebf472deeceb4633033c9edf95092 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sat, 13 Nov 2021 13:05:24 -0500 Subject: [PATCH 129/175] remove lame comment --- idb/indexing/index_from_postgres.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/idb/indexing/index_from_postgres.py b/idb/indexing/index_from_postgres.py index e1388a5b..efbb5bce 100644 --- a/idb/indexing/index_from_postgres.py +++ b/idb/indexing/index_from_postgres.py @@ -309,10 +309,6 @@ def consume(ei, rc, iter_func, no_index=False): pass else: for ok, item in ei.bulk_index(index_record_tuples): - # Is there a way to try/except the iterator to prevent Exceptions from being fatal? - # Let's try! - # - # pass if not ok: logger.warning('Failed during bulk index index: {0} '.format(item)) # We should never need to call gc manually. Can we drop this? Especially From 4708e2333cd78a5f4288f98a126096eb79b9b930 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Fri, 15 Oct 2021 19:59:32 -0400 Subject: [PATCH 130/175] Adjust settings for better bulk indexing (#190) * prevent raising Exception on bulk index errors and exceptions * Add new config variable ES_INDEX_REFRESH_INTERVAL to replace a hard-coded value. * Change the index settings of freshly created index so it is suitable for bulk indexing. * Improve logging. * Reduce ES_INDEX_CHUNK_SIZE to see if this improves indexing performance due to increasingly large documents. * call the optimize function after any indexing run where we might have adjusted index settings for performance * braindump into some docstrings * remove some garbage try: except: pass blocks. --- idb/config.py | 3 +- idb/indexing/__init__.py | 20 ++++++++- idb/indexing/index_from_postgres.py | 19 +++------ idb/indexing/indexer.py | 66 +++++++++++++++++------------ 4 files changed, 65 insertions(+), 43 deletions(-) diff --git a/idb/config.py b/idb/config.py index a2f99755..1193d646 100644 --- a/idb/config.py +++ b/idb/config.py @@ -58,8 +58,9 @@ def load_config_file(p): IDB_CRYPT_KEY = os.environ.get('IDB_CRYPT_KEY') ES_ALLOW_INDEX_CREATION = os.environ.get('ES_ALLOW_INDEX_CREATION', 'no') -ES_INDEX_CHUNK_SIZE = os.environ.get('ES_INDEX_CHUNK_SIZE', '1000') +ES_INDEX_CHUNK_SIZE = os.environ.get('ES_INDEX_CHUNK_SIZE', '500') ES_INDEX_NUMBER_OF_SHARDS = os.environ.get('ES_INDEX_NUMBER_OF_SHARDS', '48') ES_INDEX_NUMBER_OF_REPLICAS = os.environ.get('ES_INDEX_NUMBER_OF_REPLICAS', '2') +ES_INDEX_REFRESH_INTERVAL = os.environ.get('ES_INDEX_REFRESH_INTERVAL', '1s') IDB_EXTRA_SERIOUS_DEBUG = os.environ.get('IDB_EXTRA_SERIOUS_DEBUG', 'no') diff --git a/idb/indexing/__init__.py b/idb/indexing/__init__.py index 0eb059f8..53ba9287 100644 --- a/idb/indexing/__init__.py +++ b/idb/indexing/__init__.py @@ -75,6 +75,9 @@ def cli(ctx, index, corrections, types, indexname): @click.pass_obj @fnlogged def continuous(params): + """ + Do not use, probably does not work. + """ from .index_from_postgres import continuous_incremental for k in params: params[k] = params[k]() @@ -85,13 +88,16 @@ def continuous(params): @click.pass_obj @fnlogged def incremental(params): + """ + Do not use, probably does not work. + """ from .index_from_postgres import incremental for k in params: params[k] = params[k]() incremental(**params) -@cli.command(help="Index data matching ES query") +@cli.command(help="Index data matching an ES query") @click.argument('query') @click.pass_obj @fnlogged @@ -134,6 +140,9 @@ def uuids(params, children, uuid): @click.pass_obj @fnlogged def resume(params): + """ + Does this work? Might be nice if it does. + """ from .index_from_postgres import resume for k in params: params[k] = params[k]() @@ -144,6 +153,9 @@ def resume(params): @click.pass_obj @fnlogged def full(params): + """ + This builds a new full index. + """ from .index_from_postgres import full for k in params: params[k] = params[k]() @@ -154,6 +166,9 @@ def full(params): @click.pass_obj @fnlogged def delete(params): + """ + This might not work. + """ from .index_from_postgres import delete for k in params: params[k] = params[k]() @@ -164,6 +179,9 @@ def delete(params): @click.pass_obj @fnlogged def check(params): + """ + This is the standard index run for the ingestion process on an existing index. + """ from .index_from_postgres import resume for k in params: params[k] = params[k]() diff --git a/idb/indexing/index_from_postgres.py b/idb/indexing/index_from_postgres.py index efbb5bce..8998e43b 100644 --- a/idb/indexing/index_from_postgres.py +++ b/idb/indexing/index_from_postgres.py @@ -265,35 +265,26 @@ def resume(ei, rc, also_delete=False, no_index=False): # Create a partial application to add in the keyword argument f = functools.partial(type_yield_resume, also_delete=also_delete) consume(ei, rc, f, no_index=no_index) - + ei.optimize() def full(ei, rc, no_index=False): logger.info("Begin 'full' indexing...") consume(ei, rc, type_yield, no_index=no_index) - + ei.optimize() def incremental(ei, rc, no_index=False): consume(ei, rc, type_yield_modified, no_index=no_index) - try: - ei.optimize() - except: - pass + ei.optimize() def query(ei, rc, query, no_index=False): f = functools.partial(queryIter, query) consume(ei, rc, f, no_index=no_index) - try: - ei.optimize() - except: - pass + ei.optimize() def uuids(ei, rc, uuid_l, no_index=False, children=False): f = functools.partial(uuidsIter, uuid_l, children=children) consume(ei, rc, f, no_index=no_index) - try: - ei.optimize() - except: - pass + ei.optimize() def consume(ei, rc, iter_func, no_index=False): for typ in ei.types: diff --git a/idb/indexing/indexer.py b/idb/indexing/indexer.py index 44788316..f26cc579 100644 --- a/idb/indexing/indexer.py +++ b/idb/indexing/indexer.py @@ -110,25 +110,31 @@ def __init__(self, indexName, types, # on a local machine. Use single node es config. if config.ENV == 'dev': self.INDEX_CREATE_SETTINGS = { - "settings" : { - "index" : { - "number_of_shards" : 1, - "number_of_replicas" : 0 + "settings": { + "index": { + "number_of_shards": 1, + "number_of_replicas": 0 } } } else: + # Number of index shards is set to the real config value here, but for + # performance reasons immediately after creating a new empty index + # (where presumably the next step is to do bulk index operations), + # set a few additional index settings suitable for bulk indexing. + # We later set these to the appropriate config values after indexing completion. self.INDEX_CREATE_SETTINGS = { - "settings" : { - "index" : { - "number_of_shards" : config.ES_INDEX_NUMBER_OF_SHARDS, - "number_of_replicas" : config.ES_INDEX_NUMBER_OF_REPLICAS + "settings": { + "index": { + "number_of_shards": config.ES_INDEX_NUMBER_OF_SHARDS, + "number_of_replicas": 0, + "refresh_interval": "-1" } } } - # Create index only if: - # 1. it does not exist, and + # Create index only when: + # 1. it does not already exist, and: # 2. we have the environment variable set to permit index creation. self.ALLOW_INDEX_CREATION = True if config.ES_ALLOW_INDEX_CREATION == "yes" else False if not self.ALLOW_INDEX_CREATION and not self.es.indices.exists(index=self.indexName): @@ -137,6 +143,7 @@ def __init__(self, indexName, types, if self.ALLOW_INDEX_CREATION and not self.es.indices.exists(index=self.indexName): logger.info("Creating index: '%s'", self.indexName) self.__create_index() + logger.info("Index prepared for bulk operations.") if self.es.indices.exists(index=self.indexName): logger.info("Found index '%s'", self.indexName) @@ -161,7 +168,7 @@ def __init__(self, indexName, types, def __create_index(self): """ - Create an index with appropriate shard count and replicas for the cluster. + Create an index with appropriate shard count for the cluster. """ # create(index, body=None, params=None, headers=None) res = self.es.indices.create(index=self.indexName, body=self.INDEX_CREATE_SETTINGS) @@ -238,21 +245,26 @@ def index(self, t, i): def optimize(self): """ - Do Nothing. - - Previously ran the es optimize command with the proper number of segments. - - What the heck are the proper number of segments? + Previously ran the es optimize command with some number of segments. This never returned properly. In later version of Elasticsearch, optimize has been replaced with the "merge" API. We can bring this back if it serves a useful purpose. - TODO: max_num_segments probably needs to be more configurable + We co-opt this function to reset indexing settings to appropriate + service values (as opposed to bulk indexing values). """ - logger.info("Running index optimization on %r", self.indexName) - logger.info("Skipping index optimization / index merge.") + logger.info("Skipping forced index optimization / index merge.") + + logger.info("Putting index settings...") + self.es.indices.put_settings(index=self.indexName, body={ + "index": { + "refresh_interval": config.ES_INDEX_REFRESH_INTERVAL, + "number_of_replicas": config.ES_INDEX_NUMBER_OF_REPLICAS + } + }) + # TODO: max_num_segments needs to be in config if we resurrect this. # self.es.indices.optimize(index=self.indexName, max_num_segments=5) def bulk_formater(self, tups): @@ -293,20 +305,20 @@ def bulk_index(self, tups): Needs more info here. """ return elasticsearch.helpers.streaming_bulk( - self.es, self.bulk_formater(tups), chunk_size=config.ES_INDEX_CHUNK_SIZE, max_chunk_bytes=1048576) + self.es, + self.bulk_formater(tups), + chunk_size=config.ES_INDEX_CHUNK_SIZE, + max_chunk_bytes=1048576, + raise_on_error=False, + raise_on_exception=False) def close(self): """ Finishes index processing. """ - # This will allow newly-indexed documents to appear. - if self.disableRefresh: - self.es.indices.put_settings(index=self.indexName, body={ - "index": { - "refresh_interval": "1s" - } - }) self.optimize() + logger.info("Finished index operations on %r", self.indexName) + def query_for_one(self, uuid, doc_type, source=False): """ From 62fd9687382fd420dd4f005d5322e71eefa0afb5 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 19 Sep 2021 08:35:25 -0400 Subject: [PATCH 131/175] WIP accommodate changes to DwC standard - basisOfRecord (#188) * add comment about basisOfRecord * add jupyter notebook that explores flag dwc_basisofrecord_invalid * add a little more info to function comment --- idb/helpers/conversions.py | 17 + .../flag_dwc_basisofrecord_invalid.ipynb | 1621 +++++++++++++++++ 2 files changed, 1638 insertions(+) create mode 100644 notebooks/flag_dwc_basisofrecord_invalid.ipynb diff --git a/idb/helpers/conversions.py b/idb/helpers/conversions.py index 63df4607..dbd5389f 100644 --- a/idb/helpers/conversions.py +++ b/idb/helpers/conversions.py @@ -844,6 +844,19 @@ def collect_genbank_sequences(t, d): def fixBOR(t, r): + """ + basisOfRecord - https://dwc.tdwg.org/list/#dwc_basisOfRecord + Recommended best practice is to use the standard label of one of the Darwin Core classes. + Examples: PreservedSpecimen, FossilSpecimen, LivingSpecimen, MaterialSample, Event, HumanObservation, + MachineObservation, Taxon, Occurrence, MaterialCitation + + This function just performs basic check to see if values supplied are likely in the controlled + vocabulary. For example, we would not allow 'Exsiccati' or 'FieldBook' 'Plants of Colorado' just to + name a few. + + These comparisons using lowercased version seem to work, even though supplied values generally + match the class name in the standard. e.g. To find PreservedSpecimen we look for "preserved". + """ if filled("basisofrecord", r): if "preserved" in r["basisofrecord"]: r["basisofrecord"] = "preservedspecimen" @@ -851,12 +864,16 @@ def fixBOR(t, r): r["basisofrecord"] = "fossilspecimen" elif "living" in r["basisofrecord"]: r["basisofrecord"] = "livingspecimen" + elif "material" in r["basisofrecord"]: + r["basisofrecord"] = "materialsample" elif "specimen" in r["basisofrecord"]: r["basisofrecord"] = "preservedspecimen" elif "machine" in r["basisofrecord"] and "observation" in r["basisofrecord"]: r["basisofrecord"] = "machineobservation" elif "observation" in r["basisofrecord"]: r["basisofrecord"] = "humanobservation" + elif "occurrence" in r["basisofrecord"]: + r["basisofrecord"] = "occurrence" else: r["basisofrecord"] = None r["flag_dwc_basisofrecord_removed"] = True diff --git a/notebooks/flag_dwc_basisofrecord_invalid.ipynb b/notebooks/flag_dwc_basisofrecord_invalid.ipynb new file mode 100644 index 00000000..1fa2556a --- /dev/null +++ b/notebooks/flag_dwc_basisofrecord_invalid.ipynb @@ -0,0 +1,1621 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "import idigbio\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "api = idigbio.json()\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "# https://search.idigbio.org/v2/search/records?rq={%22flags%22:%22dwc_basisofrecord_invalid%22}\n", + "record_list = api.search_records(rq={\"flags\": \"dwc_basisofrecord_invalid\"},fields=\"data.dwc:basisOfRecord\",limit=600000)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "len(record_list[\"items\"])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "# known bad values in basisOfRecord (we are looking for new good values)\n", + "dont_count_list=['Occurrence']\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will find mostly 'MaterialSample'." + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'uuid': '99f1de48-d0f7-4afb-b7b9-76a23ae6b323', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c47016fe-d46e-41e3-8c21-59f4f70fee1f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a01fe901-c4db-499d-8a67-a2bcd02c7ec1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '711add73-32b2-46e8-8ebc-635265f905fa', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ddf31d8b-01c6-47a1-b476-a3a28c7b0114', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '88805377-d569-44ae-9b0f-64020d6a196a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8e1a3d6c-95ec-4bff-950a-c9f4c5666fb7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd15a5615-da19-4338-8ce0-7292488a5e6b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd9df17e5-3c9d-47bc-8ec6-83783712d774', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9076bfd6-eccb-4a9c-87a7-8f81d68a4df9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ba46e319-fa05-4744-9942-86198f487888', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4f20ea39-c9a5-4233-9e42-52f1263dd93d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '61b4ea58-97fa-4250-a41f-877f96ae6314', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e13e8257-5771-4788-9b78-c69b135aad41', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4162a3f7-18b3-4f59-a8ce-2b34d80e30b0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'eb631149-6cdb-4ba7-9354-8497b3d0e6fa', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '53c5ee3f-07b0-450a-80d3-ceb8583ba239', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e0c50a71-975b-4c6e-8618-4d454eeb7402', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a804d835-be36-4c96-8947-6d0690242ed7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '41a1a1bf-9ca0-4962-ac7a-5979fe41aca4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9768863a-f762-4c66-b294-d288cb3805f4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0f9879bc-47cd-4433-afc2-660b846f961f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0fe835cb-7f48-4aac-84de-39d6cb11a080', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '79193d88-1a60-4249-8a1a-3ca1c66996eb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b8940202-7532-45fa-a5e1-74b88ce735be', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a6efbd07-e3f6-4e40-8ce1-b5e5b40f3758', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ada51951-a5b5-4e74-9c6a-d9d2a977dc41', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '22e6a449-5ee6-4c3e-bea7-f078cc8089f0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a26c1ab9-af74-491f-b5c4-0db185a96e84', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e5cde254-655c-446e-adef-32a7497f4199', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8ab3b848-5057-4b79-a016-ecfd01d7d712', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '338ab9ab-98db-4c44-8f84-25868867aa4c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f863ddb5-30f7-47bd-9ebf-19c9edc4ae1d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'dfd0e908-d2f9-4d0d-8cb5-f09cac475617', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '21a01e7b-6cc0-476c-8b40-aaae44c1ba3a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e120e855-3b42-4b3a-afed-abae2bc23c6c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '03da6498-a78e-4669-9915-937c526d6e00', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '36581c27-0337-4f6a-81a0-00a873122745', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '26a66257-9d7d-4e5b-bac2-dd8b3cef243e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '080c1d32-ce45-4a14-b396-f0df8bf8e717', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'cfdc6e57-ecee-44d4-849b-8af4a038e60d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '88dd6d9a-7284-49b3-bc4e-171418be9a88', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '536e3fb9-3a0e-410c-af45-ef8f96d5ebea', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5f57d2d9-1463-43fa-94cd-5c1c5567631c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2d9cb617-5f15-4bd8-a5b3-627d2e84ff1a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e5b76b40-805b-455f-97fa-b4626df5897c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'be402c62-7c27-4f74-8f28-3507c03f0311', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '568ddd07-e9d7-4f40-afa5-c038369b70f8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '98fb3b91-b8fc-4f80-9ba7-d6389f9d4cdb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4e920f2c-598b-4f14-b4da-4826c9ec61f2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'cfcd88e4-0e5c-4339-bbac-3e424795c21e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1afaf14d-c483-4cee-b47b-75fb567250f2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7abef849-30bf-47c0-88db-d7badaffce5c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5445aec2-4bd6-44f9-86f9-c2e2cd02fbb8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '62bf1ca8-708b-4a3d-a8cd-7e404ac9cbf9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4a3ab048-45a7-42f2-977b-36c614dfba66', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7ef2a450-175c-40ae-9528-a1c3535465ce', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a1f6a9a8-4620-4cd8-99af-22ed81877d14', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '24960d97-12f6-452c-bf01-390b8ea68615', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd56adcd5-eded-4b67-829e-eaaa8333cf2b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '26a6c287-60ce-4d48-b51e-58b4a3a76039', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ca9667d6-0c1b-40c7-872e-494f9231dd47', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3b742fa1-2785-48db-9b58-6ec1a8d7b028', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd46f90d0-91c0-48a5-9556-1e2f09378280', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '27405e45-439d-479c-99d1-ecc3aa69cf31', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '18f4b259-2a93-4c27-90e9-af26c269d7d2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '40fc94a5-6b22-423e-a83a-d252f75d92c8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '42a4a463-e37a-4dc1-bed3-f4ea9da0f2b8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '65369b8d-25ed-457f-a540-8eeb9263c237', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1858b682-b420-4345-9761-b9b5ba68d73b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '689c8c67-a76b-40fa-8dff-fa4830add336', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bbc97e38-4e25-4a79-b8f9-b171ebe0cad9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6a779282-3121-4567-9077-f7563645d2ec', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2b2a1620-661b-4e84-9b2c-e55e1890122f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'aa36ea52-d8ff-4166-b5c3-64d710e32504', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ac22c4f3-e378-4bde-8bd5-8819b28fad1c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0acbb4d1-0d80-46d7-b5af-fd1d482ef32d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c108c3b4-c072-44f9-b22d-5a05bfde03a8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '746d726a-da73-40b8-90d3-c4f35b155e20', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b0755619-cd5c-4661-8ae5-10583aaec0d3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'dd71946b-bd8e-437d-b91d-752a91e4b67a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6e2fbd44-2997-42dd-ba2c-4440dba8ac0e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5c3162c6-f7ea-4d9d-82b5-9d22e0c51d90', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bd757993-7c69-4fc0-b456-fe5b9f977fe0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fe24bc22-d279-4243-b80c-d3cb763c66fd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '004d6e81-8320-4f3f-a63d-743bbd8a074e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a49cade9-5941-4d96-8d7f-1b93d6e0c9f4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8ad7aa26-0f28-4c1e-b927-876a68df0ab4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2bb200cc-4aee-4536-8723-199f01a1c3fb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bb6155c8-6bde-4783-b21e-654214975ca7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '71ad089e-8acd-4ce9-95a3-951dd3b8bbe8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '77eb383f-fb82-4901-854b-ef0650b12d51', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd5f89cdd-c63a-4ca3-b633-e333a3cb25e0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1ccc187d-ed37-4fa3-95bc-41a4c1fa895f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2fa2ef13-c9f4-45c6-aecb-b91c30c7e3b4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b909ae22-bcf5-4b6d-87c6-b000859765cc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ab15c459-8c3b-4a5f-8533-9b963d010079', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4241aa3e-0ec3-4b55-b9a0-5d00f5b4ab53', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '25c2eabc-9edd-4e15-b85a-22910c866177', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '969e0729-f747-4118-b79c-7a10f8fdc3e2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5edc4d29-e8a8-4b16-b54d-3d45e2bcd1d8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fbe87750-d2ef-4dac-9e22-e1b76f86c971', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1a119ab9-b28e-4fa3-a675-ee583c9c2e8d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b8287927-e8cc-45ed-a621-d36cb5b4186e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f7edebda-b7e5-4f74-9025-c487a9fe2c72', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8d8cd4cc-3b32-4197-90e9-29a94d87bfe8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '02021691-2063-489b-b48f-5d9e20554d9e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4059f23a-5837-40e0-b414-491a9c92cee2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a41f24af-6b77-4462-9f46-d8dc2a081049', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ebd003db-df5e-4c41-ac34-7918b9779d15', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5f4ee792-0023-49e2-b82b-2603c7f2344a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'de603546-c648-43bb-ba58-f8a94cf71500', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4664d311-4629-47dd-ad36-b0c801fc3fd9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'edbc449a-783a-46b0-ab27-e853a0a08c2b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ab42201a-3319-4fd7-89b3-704fe2cdb552', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f03a88df-0a45-44d1-a75b-e4b8200a2f59', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '36fc3e5d-ecb7-49a7-8145-7749bb0531ca', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '93df51e6-3f9c-41a7-9aa0-1b8e917ff000', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8e0c2b97-d725-4513-847d-303bc68615e6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bac09599-1b50-46ba-8c78-4674159b7186', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd5d901d2-e89c-4d12-b7ca-5f80e9159481', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9b849f00-c5fc-470b-84d6-3cd7af22f649', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bab5b08d-7321-44ba-ba98-3749d3202307', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bf813651-c243-4e2d-8c80-8c5121860b71', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '305cf80f-b42f-4af7-b2ac-9cb993a6e712', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '99e8b18b-65db-424b-8967-3f43919876f5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c038a4b6-815e-45e0-a823-eb34ba18d237', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6267c56e-1da0-4805-9f21-7d63b3ed1b45', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b5f73ede-034c-4974-b6c0-dcde1cf7970b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'beeb1b18-298e-4cde-b876-9b3cd0f1f12d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '75d7f00b-c3d2-4c18-9d92-4ef1f8638e5e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '741f2fac-e413-4f50-8245-f3086b112435', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2ef99c48-3a51-4cc0-93c8-2f8de28c95cc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Plants of Jefferson County Open'}, 'indexTerms': {}}\n", + "{'uuid': '22afbdc5-7f1d-4c01-8997-70827ae86197', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a4e34706-fa9a-4907-8250-68b18d8825d4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8b9c3891-44f7-443a-a0c1-54c6d613341e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6f7c4bad-6167-403e-b247-5cb08beb4359', 'type': 'records', 'data': {'dwc:basisOfRecord': 'materialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'cc0c9d37-b53a-4225-8740-dc4fb19cd8d2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '90215558-9300-456c-bcca-a8e2d4cd755d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ada88cd0-5505-48de-a889-daf36238f29e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '61573df1-2240-4848-a41b-d4b49a62cc68', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '65fffc10-769f-43c1-93e0-8286d3476625', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd214d0d7-e02d-4afa-a4e3-580666ad77e1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ac294322-3f3c-44b9-ae9c-895255d69959', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5ce08e13-d0ca-4dbb-ad05-603857153ca8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5cd86e6a-f412-4b17-b08c-17c59bdbe2ca', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5f502b1c-181a-457b-87c2-869e850af2b4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6f86d67e-64fe-4d53-8929-4d41950e69eb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4e0c57c8-35d5-40c3-a287-12084549b8a2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5d9d14cc-19e1-4ed7-830e-c6f9492bd9ca', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5d2619b5-d681-475b-ba7c-6514dd6728f0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '96e3b07f-afa8-435c-b37f-1c1b22e0e9f3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '243c2ebc-2042-4741-81f9-0af9da7b9f44', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b08c8068-484c-42c3-9b44-9d463df005b9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b416f2d1-a115-4b18-bf9b-7b41fa257399', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5b9417ff-8071-479b-adbf-4ac829d90785', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7141939a-c459-4df6-b64e-a2e93714a2d6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bd8ad915-a8d7-43e5-b4de-6d770aa3a652', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '25a242e6-a0ae-4bc7-8425-a6dd221f0cb5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '199943ca-9b67-417d-beb1-0f4efb01195a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '63028785-a8e9-4d2b-990a-c77eff0883b6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'cf03a8e1-34ed-4103-9f13-4c751a1b2055', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '64528229-40c8-4365-94d7-deeb6ada62a5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1f90075e-e1a6-4ad3-9630-84b5fdde5fac', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '448f7060-987c-46c7-9e1d-7fa7493d8a46', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '43189160-a965-41fe-a798-11b6e5900b3f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0cfc3069-c1a0-414f-98e9-4950b80b0781', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd30466a3-5cdf-4c9e-a845-f2f9b4a72510', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3b11bbb2-38f5-4155-93e4-1d8dbb6c26b1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c48ddd7c-94bd-4c87-bfcc-3c12d4ac18de', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '75f55575-dbb7-42cd-9e1d-c1ae58cd7958', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a38e8101-d841-4ca3-b623-3dfceb50c66a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a74b1b99-b164-43a7-ac46-581b1af54df4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '56187c6e-3e26-4723-8cc2-15ff620e9052', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c5094f08-a7f2-4149-93f6-c547523238d5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2519e1b7-e13b-4a1c-bdd5-8dd28d133a67', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '845d3ed5-e36f-4087-80a4-e6e95c33f027', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '750fa850-7d67-4a1a-9c11-376862081158', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6fdbc4cd-c25c-4690-8fd5-b66c53aa673e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '79137250-43a8-469e-a725-2b7bec6671fa', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '131f76cb-0fa9-481d-9a90-34b4bf0889ea', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9dabc0d0-7f96-454c-ab69-efa9df12beff', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3ab93e4a-1119-4dbe-b8aa-c7b1d39b7ee6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f5fa2134-09ba-443c-b591-c7461885ba8f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '904bb02b-e8c8-4c80-9a95-e1a127daab69', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1df9dd50-84d5-43f2-b903-b4ab4a7c4863', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7a2375cc-a71f-4c36-852c-1cc0dc8b0411', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c9c9fd21-7a8a-4055-ba6f-43ebbf0b09ff', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ad27ce34-7372-41aa-a606-b630dfdf5e8d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd7f75d97-6794-46a2-9f6f-7245dceae522', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b7b86c1c-4328-4bd2-932f-a55506fb7775', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '41b03e3b-2210-4189-b709-9510e897b2d3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1ab818da-c0c2-4cbc-9255-652b3688c22d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '423ccc8c-8975-4980-87fd-36cde7f13f3d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '67844dda-5ec0-487e-b880-85e3fe62c1ef', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5363beb3-7e10-4e37-98d3-8d1d942a5e1f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '011cda5d-183d-48f1-9d50-70d30647be2d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4800fafb-dca9-456a-9d59-4543ac13a74e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3efa2349-6311-407f-80a3-b32189842fda', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6d3a44bc-3a87-4552-853b-906537da3ef1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7aeccc5e-426a-4610-a570-fea4f2d6f635', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9e7f8d74-f3c7-47e7-b0c3-fde451796558', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1b9a8880-a5f7-403b-a52c-a712e75a3b24', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '473f64cb-0bb5-4baf-bbd2-baea7c79826d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '803638f3-b6d1-4657-9be8-7cc3ad0279ad', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ffd09a6b-a5e9-479a-86b0-4a9509844e06', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'de8c4f9d-f4ca-44ad-96bc-f2ea039a1fbd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'materialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd1588556-0162-46dd-bfde-fa5e6b6cb7d3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c930c984-a035-4e02-9d7f-f706be263525', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '88e874ad-b609-4893-aea1-2ce0e28e33e4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd9ff1a32-2c9f-458f-b74a-8a0ca3fbd44d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '58e03ff9-36bc-465b-bb44-6af5f1cd45cf', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '59e6bf3a-9938-48e5-b058-20cac0f0ae4e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8322704b-836b-425b-b74e-59549484a2a8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Plants of Jefferson County Open'}, 'indexTerms': {}}\n", + "{'uuid': '6ba6c143-ba93-4c8d-a1ad-c918c629a32f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '75b006d0-eaea-4a1d-bdd7-2692391184d5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '813b7e1a-81a6-41c3-aaac-27c002a39196', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f31a1b80-48f5-4522-8ad1-60ebf9a47dbc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b3909e09-023f-4e3f-9517-a25f91e5f0ae', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7709b0a5-86da-462e-81e9-cd44b96b817e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '620085ca-654d-422a-9252-dba6d5634db2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6cf4f798-9d74-46aa-8d9a-ed3bd9656cfd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4a339c41-e9a6-47c5-b855-9c7233adf9a1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '474ba38a-e999-4676-ae78-8732bf4170c0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd95232ef-83ab-4863-bc43-8fe7885fb72e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b70b4770-7104-43de-9193-6fe74ba6168b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e5608277-2b91-4e30-beba-a9519659337f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c4342e25-41cd-4939-9f4f-2a8d155615c5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a19a842f-fd9b-4c06-bd14-652408cf5d4a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Native and Naturalized Flora of'}, 'indexTerms': {}}\n", + "{'uuid': '03f7094d-5778-440e-b5a9-32ac3b63d158', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '92ada32e-6912-49cb-a6d2-453d24fbb316', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '953f4b05-b805-4965-a624-37c196ae4fb7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c38c135e-897b-417b-871b-b315c0df0ed5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f1e69410-8db6-4df1-868e-7e7e01402290', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1d1c82a3-d604-40c2-ac61-34a931c89cb0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'de2156a5-444f-44a5-b6bc-adcf5381f743', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fe3ee00f-56b8-4486-8810-74268ef34310', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c5095841-537c-4c30-abf5-62c40c6afb61', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e7cf068f-997a-48a5-b31d-06d03eca9618', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'dddf7a87-d8ae-4a7b-8ead-07996f9d454e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5cea2bef-3ebc-4544-9b88-6cf31d8836d4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f7e9fa92-14d8-4566-9f63-21077eda85e3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7006dfe8-8125-46b8-b80a-779ed76b7a9a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'cfb75f04-2a68-44fb-9672-0f033678cd99', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '900e3330-9df5-4ae0-b23c-16b38f8f1971', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '081d93af-b42b-4beb-85ac-0b6ecb99ff40', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f1cfcede-9be2-47fd-ae4e-12f0c0788ae2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Plants of Jefferson County Open'}, 'indexTerms': {}}\n", + "{'uuid': 'f226e6e7-fe8c-4b07-bd36-2e46f03b7930', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Plants of the Mono Lake Basin'}, 'indexTerms': {}}\n", + "{'uuid': '6e111ea1-2abc-4c7d-a811-6e11ae38838f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4e107a3f-fc82-4553-8160-a795f6a20d29', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7a694218-832b-4217-b58e-93d0154f6d66', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '43aacfb0-d27f-4427-8fca-e364c7984b69', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '024e912a-5681-4c6a-974e-eebb44be486f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e3461dd2-d04a-4933-bd67-f3ffeeb85de1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e9f15a27-1c12-407f-a4f7-9565c90d557b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '07e333e5-5ebe-4672-8c5d-b6e51bef7a1f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '873ce776-c67d-4a83-99f6-699ecc7866f1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '975f169a-add1-41b5-a7ba-385e0b6574df', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '160c7012-7985-431d-aad1-c92d0791250a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c63d638e-4c85-400e-adc4-369b37f29e70', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9a27763b-c3bf-478a-866f-c61167a10737', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5e1e855b-12f3-4b86-8633-fa4e57430784', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '38968dd4-446a-4241-b0bb-1e82497ee956', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '19c45537-f441-41f8-9c4f-c03b3b14b7a1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5c8b3387-532a-4659-9950-87d7fe395754', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd5f31146-d9f3-4409-a50c-96c3ca306b27', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '59cde993-f70e-403c-b16e-8810e7a6dab4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'deb95de8-432d-4f91-8ca5-a94fcec2bac0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4e0fae64-4dfc-4941-a282-dbb782645063', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4d568f73-8d05-422f-80dd-a61154380dcd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6d497fa4-9762-4480-a053-ad1d28ac84a2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '26ca6dae-cabe-4f61-93c0-1bee0279d64f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7957abf8-aeaf-4f77-839a-0b39a523aad3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ff010503-63c7-4dc3-bec3-a8c85b4a579e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f8f08b5e-e004-4707-a84a-50fab4210d30', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '56348e6c-f0e6-4dfe-9976-bab0c4e0e2d2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a36e22f2-5dc0-4445-a9b4-6e9a0da43b7e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e0fbc486-363a-4cce-840c-2964d5dce618', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '93b15410-084d-4722-a2d1-b91236350e7a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2e6b3c86-1c29-49b9-abe1-574bba4ebf19', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '40650091-1866-4d8f-ae9a-c34e0c2f3ad8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0d1371cc-5065-4c47-ad80-18ed95285723', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '79e3dbe4-7f38-4e12-95f5-5907f0c6ee2a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bae1fb20-ebe4-4ef6-ac5c-fe8d441aa24f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'cd46b7b0-988d-43c9-b0c9-720cc6f5ae37', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '30921047-6bf7-4772-9873-0ba9ff112a11', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '39fd35f9-35c2-499c-8561-3c461be1f023', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '482e65a5-f6f1-457d-b00b-5e394da6bf88', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '750ab66f-ab2a-4856-8b5b-46bcca4c71f8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '906ef690-e2f0-480e-962a-ff5d5c00eb95', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bfca6a29-2527-47de-ac75-7cbbd13853e5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e28dfb40-eae2-46af-a695-82358134beaa', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '35c15dcf-9b2e-41e5-8e9b-684048baa060', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '78374397-8a3b-4846-bf5f-233772b0a6a1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1e334c28-69f3-4e5a-8894-30c80720aee2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '365dc398-e807-4a9c-b708-62cbbb7ecc4e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fae0ef5a-5dc7-43dd-9d3f-c7233c3fd3cd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1e392089-f72c-4948-96ac-c502bbf791c8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0cebd33f-3a26-4e69-8223-507d98f9be05', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7483a96f-ed68-4fd0-887f-ea07e442ee73', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '53c65c2a-ef2b-4567-8b46-1267b48c4023', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6d6b3a65-cddc-4dc6-b29f-7b8807c94203', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c9a88d8d-71ae-4a01-ba8a-310f559ebea1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd6d33006-b910-4ed0-a410-e192ddd45743', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '05bef81f-ece6-4067-bdb1-c4428704dd4d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6d78712b-4d21-4820-9792-1fa6943d8431', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'eace66d4-fbcf-4220-ac27-0b0ff03eb985', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5398b83e-2ba7-4169-b12a-f755791b759d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '69bf09ca-8c55-48c3-931a-d33426aa167a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd572bb82-3d70-49e6-9be0-571eef12b4c2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8015a262-7566-4726-ab59-73b31584f21e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e6b253ca-c998-476c-9f0c-132201a5f1a4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9a8e7567-9115-4581-8ce4-def54942aa4e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '57327677-210e-4c6d-afa0-e8c574244ebb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0ab0032c-efcb-424a-83fe-66b85641815b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a9ddc7ec-a138-4d7a-8af4-d8dcaa88edf5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '20af0301-af44-4d9e-af60-2c4b0cfb92f2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0237a2e2-2756-457a-8300-e5fabc4ff94c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b817525f-72fd-44cf-b029-e68fbc7c4268', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ce9ef0b9-56ef-4c78-b11c-048c31102b7e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '59b5f77f-91cd-48f7-9c0a-027d9958de28', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ca15c135-988a-4129-86d1-6b7c1887a825', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '80caa0ec-6a1f-419e-8846-e710d3fa9ba1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1dd4fd06-9ccd-4a1a-a0db-c05095d60461', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9207c3de-07cf-4382-a1c5-ce9314dcd9e4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '81838331-5d6b-49f7-8866-a99ea0a409f5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ca07b702-79f8-413d-b78d-b5d5884a9134', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5fa79023-5497-4736-b981-0c2afe450bb6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8d915591-9f6a-4449-ba32-75f28f9a0e72', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6d7bc6ee-b2a5-4f7a-9881-74278b10e8b7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '868bd2c2-5f04-448d-91e8-99e2d3edb61c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f4e57304-9e94-4fc3-94a2-bba5730d205c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '66736c64-dc06-41f4-b31e-acefc76376c4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '87f4b8ed-24bc-48c6-9f8a-d261d8a6fbf1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '62c86e32-8eff-4616-b166-bc76ee30fd0d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '74d37fc5-aad8-431b-8f00-b337f585a4cd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '40a62597-c058-4756-a26e-8379a74e4e41', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'dd0b5520-9628-4e42-9a89-29c5c50914c1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '23bf1911-b3d3-4824-a7fc-da719eeb24db', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ad7e8ade-998e-40e0-8a1f-e936d3f1dd61', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '228301db-6d1d-49f7-95f3-374a655a4182', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '30894424-112b-4621-98a4-21897d2458d0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e2115127-53e8-4dbf-9e61-ad06ab967b8f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bb4a9669-d303-4b82-a7f4-d6d374ce7cfc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4acf503f-3cb3-4878-9c62-757289dfd4d7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7f951477-a5c1-4c89-aaf7-c18683c62db8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a304c19a-2b80-4593-8352-ba1c04f8485d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7cafd772-17ad-46f5-a24e-7b62678d3eda', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd8e2d45c-9f3f-4c4d-8cc7-7bde1d2ac7b1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6327dc00-81b6-4d1d-bc9a-444ae2d402dd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'cb5c12ae-77e4-42e8-bf5a-6a10e1322d66', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ccad0a31-dcf1-44e7-8536-c6fcec379ea7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3fc0166c-f05e-4bb7-a633-13036cad2b70', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a946dc38-bb51-4552-9df0-e8684fe0c95d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8464b68b-128f-44c5-bccd-12b103aa71bc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '55ba20c9-a847-4ec8-8ee2-e0675d7a6e58', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Plants of the Mono Lake Basin'}, 'indexTerms': {}}\n", + "{'uuid': '2f8c7ed2-2db4-4c24-8369-6de760f31f7c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Plants of Colorado'}, 'indexTerms': {}}\n", + "{'uuid': 'b5b9dc3b-fc9d-463b-a60e-6f599934c585', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Plants of Jefferson County Open'}, 'indexTerms': {}}\n", + "{'uuid': 'd511400d-1a08-453f-aef1-98c641c7e380', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a1d4c96b-4b7c-4157-8df6-60a2e468905c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '107286d5-dab0-4733-8315-e648c06c4680', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e6b69584-4fa4-4026-8125-7e298e29baf1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4a2c6003-3d9f-4d77-bc70-a8d3f42549c4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7e71dbb3-1c91-4920-9461-a54cfc02debc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '06654f3a-e1a9-49f7-bf05-2687115144d5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '057e5e86-f8ef-44ab-a7d0-65097ad58188', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '215f4947-08f2-44e3-ae88-e2b7f065f53f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '88f767eb-beb5-4a5d-92ea-4eaadabe5242', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0facfe08-6f49-4d93-aa92-bf7f370833f6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '87bbe1e2-8c79-4ed1-8221-9d1921679db9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a2197fd1-52df-4cc3-9c6e-226bb051b2d7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '063f728e-1f95-46c9-83d0-24f3ce3bfdd9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6395ce09-0929-42e4-9b4f-a63ff8233dbb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a157eb9a-3189-42da-851a-428f378a08e3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1d1f5059-7f57-471d-a4d7-02fcb8ada85c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '87ca51b9-499b-4c39-aefd-5b199294e61e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'abd70157-d2a7-4604-b183-4c833d90bf1e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9f7beb00-75ae-4437-b6c8-78c3390ebfc9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9340d665-a403-483a-b6d8-54485480c3d9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '75acdb23-4893-4805-b663-5d5f957565e2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '04350fec-57f4-40dc-8db3-5d3c7096c478', 'type': 'records', 'data': {'dwc:basisOfRecord': 'FieldBook'}, 'indexTerms': {}}\n", + "{'uuid': '6a8e05c8-429e-4c40-a3dc-83b1b73e3ace', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ab3a23be-ad37-42c4-b1d7-b263fa64ee3f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'daed60a1-8fad-449b-ad4e-5fd8d7a5e18c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e3f41f9f-dbc3-4909-b6bd-8e1a05d4332e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '03d04d3c-6df8-492b-8ffa-53aa8bb68579', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '245df028-1004-4f6f-a77f-b5a9fe5b584c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '98da1d25-0ded-4dac-94b9-9caa3f7c7dd5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e9b69aa1-df2f-40e9-a883-e4317e1b16d9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '13ea8cb2-9532-4579-9c4a-23e239628f1b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '667f9fd4-b712-47f9-b053-94471e5acd9e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '83a26e6e-aab4-4e74-bc80-4b2e7b113acc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9b8432b3-21a3-4607-912a-3ce8ea9a57d2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0b2c6ff9-fab9-49ad-91b8-1f0fd2359608', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2be85215-9043-4209-8282-43e548dc5248', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c34d138e-1992-4ca4-9f1a-df43ec8bf837', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '23decd0f-aa41-4c49-aa4f-ece460a4f44a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b9745c5e-c9db-4376-a9e9-e3019951ebeb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '66bdcc9b-0315-4646-b2a6-59641c0fcdf7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3fde7275-4bf1-4856-8cb3-c2a5c195956d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3a619673-9d72-46ea-a254-e2dd8ed39ae1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '25d3caa6-0a17-4def-91ee-acb937c6f41e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0b157e7c-19d7-4d0e-b4e9-6c66bd193a4d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '00f6fd7b-b847-4c92-bc57-d46ac358c30b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'materialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0af48edd-95cd-4377-b2f0-d7740a8c6f81', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f66392e7-9b0c-48ee-960b-74f77b2fa06e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'eefd4afb-83cf-4127-81fd-bd6bff05b20c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6ad948ec-bf56-40ed-b403-409c693e3532', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ca73353f-5e2f-4660-b54d-7ee0236b3904', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5991a81a-d3b2-4954-b846-1288ddd2a630', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '499aa799-103f-4b24-85de-13e6110fbeba', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'db82c907-1f53-4cd7-adce-1ea6b481ff5d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6fecdda6-aed1-4611-a5ef-f18e53429bdd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd10e0c76-2651-4e22-b07a-1af473d542f9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3373b084-5e04-4a08-8f8c-2a449f3d8841', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '646bc6e4-db1b-439b-942d-cec8341a3747', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '93771540-83ec-4e4b-a5a2-54422c02f348', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fb0b07d5-11dc-4db0-910a-c80aef958b26', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fc1652cd-4937-4dd5-ba42-03b78f0a67c8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'aa7948ed-5f14-4cd1-8ac3-4b77e69ea394', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5ca1e429-5be2-4c95-aaa7-e71cfccf89aa', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a0176adf-8c55-4f0d-aacb-2db34ec4c84c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '13ddf5c3-30a1-4f85-9b60-cb117b11e8ca', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8f34628e-f92b-409e-91b9-80a95e4d9c27', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a6d337b0-9f15-4b88-a85e-46b869fb2247', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ce7ecc97-7adc-4573-8084-53dea5f33386', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5fc0304f-8872-4884-99d3-6f2611316bb9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c2812156-99a8-42c6-8f88-0a1234024ee6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1eb11a2d-2381-420f-96ca-cd9f97f6f69e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '35043a56-73e5-4d9c-b5b9-3bc986a6019c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b6715c68-5552-4acc-aa8f-51a64ab60af9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': 'd4e51055-d8bb-4f9d-87cc-0b86b9e03049', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b6dde35c-7725-4592-b988-280f21be01db', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '073914ff-df37-40e1-86d9-f00d9a53d5c9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4ecb90f2-6bd8-423a-973c-66830c88473f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'df2f5d45-536e-4eca-a322-6308bca9ac21', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'daaf56e6-3a69-4100-bd05-47e6c3c2ec6a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f17cc107-4c8f-44b8-a2af-69181b07361a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b6d6e35b-9a92-4ba9-8545-648e64764052', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e38d0c4e-1a72-4f5c-856e-2c5bad8461d3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '770e8c96-1f1a-4555-bac9-af5b87a6194f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '58c5d7d7-2299-4505-a42c-508ca18f4739', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c293f62e-4ad7-4dcb-a67d-7fedcd0ea1bd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '02c81000-14fd-45e1-b640-3a491b283437', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd0d71ae5-b383-406b-be37-126180dc927c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0efd9ce2-3027-4e1b-9860-7e81d0427157', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'daa500e6-af73-457b-acbd-257ee78d1adf', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'aa30ab6f-e9e8-461e-b454-62fcaedf47b4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7bb24864-b4fd-4ecf-a52b-4c324298d0cc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c9130ca4-6512-4be6-ab8f-d2524d4d8b7e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7a65d71a-3f5e-429a-aeb8-29f842083ac7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e374b04b-a7d3-42d9-9e81-6160dfde678d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c59736e0-6454-41b9-9102-b40e4905903e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9a3cdb6b-7004-4e7e-ba62-2b86778bd70c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7fd5d418-5525-4db5-bb1f-d040db016823', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '76e1d566-6153-47ea-9d4d-deff2985c05f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'aad9345d-6abe-488a-85bb-f3b3d8cd8256', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e9ba3de7-913b-4496-b6b2-b8a9acebd4b0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd7273908-436f-4ea0-989f-8e9434e3936d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e95139dc-eeb3-48dd-8595-a3f104baa167', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '92ef58fd-9f6c-4339-b662-ff960d5affa0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6c09e4db-5b08-463e-ab65-d582413b5d6f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '96945915-a9be-43b8-b930-eb06c8a25054', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '36bf597a-ac32-4dc2-abac-0f7a9aa17627', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0a596936-5ddb-4f9d-ac53-5e9aecdeabb5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '19a576fc-3406-41ed-87b8-08c618fdcd95', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '527b5c50-4407-4758-89ff-87e310938e08', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5cab1e5a-ea2c-4823-883a-1e527a8ba077', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b3b04318-e6de-4f4f-9391-ff5457796ee5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'deb24539-a4bf-413b-9058-79f3f86bc311', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1835569d-cc3a-4c07-beb9-79a6b99a693e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '26eb5943-954e-4581-9d94-43e382b3ac5d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5f822517-d5fa-4ec4-9555-ce7a03e3d51a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5f460caa-eb21-4836-b428-c3b3d6666891', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2189e3e8-a941-432b-81fd-e4a2f57b1833', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '671ffcae-be95-4ec5-a585-39f9f1a78aab', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'be04b1fb-8c07-4d5f-baf8-30be52587449', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f6797d6d-030a-47cb-9184-a8686bf5da20', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f080500e-d0b1-4a2e-936a-a74d4870cb1b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '546121d3-04c2-40a4-9cee-161aa289d087', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6d8f5328-9f17-4412-84da-03d4978c4531', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2f414650-3c7d-4b09-8fd3-5bbc895b8357', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '372b2b0e-518d-4564-8bf2-2efff898d36e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '863d2712-f1a2-45f2-bf71-7a743a19fe02', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c84c5d63-cada-4b98-8c53-5fcb18a9d03b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '14ee96de-51a8-4e30-bcf7-67549d4276cd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e4c5cc61-657d-4ca8-81ef-b10ceb78995d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fc1b56ae-ae96-44e7-9b91-869ff2b9d3fa', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4eedda93-3001-40ee-a77f-e35fb7e785c2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fedbab8f-761d-484b-8386-d3a1ad5cfaf8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '50da81b8-db81-4c4b-a4f9-aaa61f512dc6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ecc92afe-95f3-45c1-b38e-91de2104a313', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0b1b73a9-e9e6-43b2-b75e-1921f834553f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '69d377c0-d7b4-4d43-9608-e7bc26048c9f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '165a8ea1-35c5-4709-938f-0ef404816cb9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1c2aeb15-11e1-4908-b7db-cb4092d0a2ce', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6ab664ee-a243-4ce8-a2a0-cffeb39d9d92', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7ec3fc69-c0bf-4d75-9916-251ba14d3040', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ae1cb2f2-3d4f-4227-9115-11eb51d26eaa', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7ef4968b-fc4a-4204-b493-885b82e83944', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bb7146ea-3855-485c-83cf-b79258094879', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b5cc3ba0-a9ba-41d8-acc0-b38c715912ec', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '27b3bfe6-05be-41eb-b2ad-0128e6960c7a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '32d75e77-ef67-441f-aae4-c0979c8a8f5f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f82db9f4-af8e-4838-8120-24a457679a68', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '325dbbfc-7c65-4060-b606-4c014aa4b6df', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '192f07a3-be72-4e58-a1c8-c7fd0ee344d9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd03b1164-e3f4-4422-9590-cb8cf2206072', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bfe5dd92-b79f-4cea-89de-246b5cd79f28', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '073d012b-5b08-49c0-9f43-cc93ccabe35a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd52887ba-0b71-4299-95f5-745400891148', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9c7051ca-3312-4cf2-9db7-be232173cc88', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'dcc54f75-77b0-4243-8b02-ed0c59d33c1d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '96b66946-342c-483c-a2aa-064b00a8102a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8bde5cee-0a0b-453c-909d-0b9d40097fcd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c527323b-34de-4054-afaf-fc9166c2a418', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c4500686-01a6-43c6-9d95-5ee1ba3d2309', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e3d16ee2-c7b5-4fac-921f-0255a4f8fc56', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2f09cfd5-cc6c-4913-93e9-3fea3eecaa01', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2d71b6f3-d9d9-4848-b4c3-7ca616d9e0b9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8ecc7856-c682-47d8-a523-3382dae750bd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5100f41a-4c12-4ff2-aba3-d3a0c76266dc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b2532df8-a6c5-4957-b75c-e07a33cd88e3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bda2384b-5d92-4586-9f1e-cd0051acb078', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '87ddf22c-d198-4fb9-a7d9-35aaec70d919', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0517619d-351a-432e-a192-ea32884c5794', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'de520b5d-b217-4118-9c07-f51be5fa58e2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '44864fe9-1ab1-484f-8038-6aa3c4300bd0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e6a3495b-d22e-4cdf-b96b-fb4fe4a613bb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c4140bd9-c624-48d5-8f09-10e2b6e2a95c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f6404939-7d2b-4019-a146-0b2eca6fd623', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4ddb30ad-a878-4ace-8d3e-14274668261a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c32ecaa8-37cc-40ea-b47a-0424ed95adac', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4f3c8840-1bb8-44f7-a5c7-d13e862a001c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '279dfcb7-f5f8-4450-95e4-e079e51daf8f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5f0a9aa6-e7ba-4474-a443-19371985f53b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '27d37b98-4869-4462-901e-49742f30afa4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fa3c4fdb-7249-45a2-82b3-fde8140f0060', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '14ba7713-aaf4-4530-ad46-bf610f00f3b7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5869c150-4b66-412b-bc5b-c2cceaf4765d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f24e2dba-4142-402d-999c-7ddfb64d5523', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f523293d-a367-4bd0-aaa5-32d76043fc1b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '949d8cb2-b9ef-41e2-9a3d-034b531e0476', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8ed5f372-7d7a-4f4d-bda3-c560a52c19f4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '12915b6e-8590-44b8-9711-7822cfb3df10', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2d14626c-60f1-4433-b3c1-b83ab35b1983', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '67d8d8d3-bccb-41c5-a2c8-b64fe321fd0b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '82f17c79-1967-4dc8-b5d9-9fe2627f3f5a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '849d758c-4878-42f0-a0dd-6fb03fae48d2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '14fd1917-a2ba-4136-8f77-e463ec653c7d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fa4472c3-1608-46b9-affe-c948434080aa', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'cfe34650-83d5-4c0d-9999-d28fbdff35ae', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6bf113c3-e9cb-4d99-aaf3-271b10ab16fb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7eed4fe3-ff88-4229-8b28-c5bd7698fd62', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e4b58e7b-100d-4831-b902-5c7a477954b9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '56f46253-d0d3-4ae9-9082-5473c09bbc94', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9597daeb-4122-4472-82ff-869f04e21932', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '28a6c3c4-7bb3-4585-81dc-2b382cc9b567', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e57f4386-5b54-4ca7-b6b5-4040fd0ed342', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e7d79cbd-7282-4e7f-8d23-30952f088d60', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f28d69c2-a167-42e9-a1bd-7a670fff1282', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '48189456-4b93-42b7-a6e9-3ae450f9c700', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1c1d7c93-4a33-4efd-a313-e976f3f9c57a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '01f51bbc-00f7-42e1-8c49-fa4d742b280e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '61bfed79-ead4-45be-8e61-428526bdbde0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd8bf0131-3a06-4a03-84ba-0d0982ebfb14', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '559e6d5e-db30-4669-a9e8-2c83053000df', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2ec86656-e301-42dc-aef1-5a57d9048f40', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e5eb544e-0ba0-45bd-88e7-03c35b0bb250', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5116d7fd-c7a8-496f-ba2b-05ffea38c1e0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd5b60b91-f10f-4645-ac34-f2307702b792', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4490118f-9e94-4e49-9b38-8df4b5e68dba', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Plants of the Mono Lake Basin'}, 'indexTerms': {}}\n", + "{'uuid': '102b8231-f302-4189-9112-b12fe7c48099', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3ce45848-c4a3-435b-94b6-150a5eca0607', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '766b8ab6-896d-4483-94f4-8ec750806359', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f4df9f8f-ceea-43af-864b-136776f9858f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5d19798b-0e53-4d08-b90c-e9d884919b7d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '97393d38-e8d3-4c47-8dfd-868237e236d0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '424e08de-c8db-43fb-ab58-18d3cee12a02', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c048caab-eb9c-4cfc-89e4-8da147c1dc5e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ea52241e-776e-447f-b3e2-9ddde18d72b1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4c9d14b4-a545-4215-93c4-36ce6621adab', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '532ec028-bfb0-4368-b168-6777a249a294', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c6c783e7-6fbb-4320-9401-488f35ad24da', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ffacc994-422a-4d6f-8127-22969245614a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '600d41b5-d50c-40fb-b0b1-f403de78f7e3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a0b45405-ada7-48f1-918c-9e845d8194c7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9ea23cd7-413d-40f6-aa73-aebd75c1224e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fd285607-1992-4331-8e45-30342c92dbf0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'dd1a5512-cb15-49c6-b64c-ba75f2f0f50d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b5d3a118-032e-4d63-98a7-638977051297', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7d1b41e2-36ed-4df6-b1e6-a9fa5a4c195f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5dc444e0-4ac7-4867-9d3a-4ad80f2456ac', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '893a1a8c-64bc-43e1-a2ed-8c4264ec41cc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a5b7a18e-fe3f-452a-ac0e-6633ffde082b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '02488156-1912-4ce9-b002-cb0693bcd3f3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0f69f6ea-4ef3-42e0-8e2a-b30455a7f5a8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c7b1c5d6-5361-4858-81cc-d9c3d4ef91ce', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c66ee17b-703e-45a1-87ed-58046e2bb226', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ae2a2389-0648-4cd6-a27e-f077c131c353', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '887daf1a-288e-4f5e-bc8c-f0a6a2483852', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8e275982-0c3b-46ed-908a-5f2483eca14b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c503c316-d9ae-4232-8d8d-b1cb0966b01f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '862f17f6-eddd-426c-9424-c4122810ae8f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '32ad1008-3bfc-43f2-8f87-fff48fa5a855', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bcd15d2e-5d6b-4a61-ba92-1a0707ed8fa5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd48da5fe-52c9-41bf-a420-2cdc5dc29f7f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ee95a355-8f60-48da-9035-760e0efe7790', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '40456a20-0f7c-4ec8-86e9-333b118e7e88', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'eaf27b6a-a750-41e4-a61f-58b05e873852', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a0523c6a-3831-4a97-907f-17103b54f30a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b11f4137-5079-455a-af32-7c498b933107', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '92393dd5-3171-46e1-8bbb-7f735312ea1b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '78615dd2-2796-4397-bcc0-bedab130c356', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e45b5087-2584-4217-ba88-971980277650', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '755cd0d4-7acb-4f32-8073-1a64e94753d3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f12b1dc0-f42b-4a71-a87b-b3056ce74b76', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Photograph'}, 'indexTerms': {}}\n", + "{'uuid': 'ab1bd07b-2f72-4e31-b426-1d8ce42727f9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '32ececfe-ae40-49cd-a5cc-3b50e99b0763', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f21fbcc9-af3b-4f79-8dca-d1aeee6f302f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b66afad7-0c6e-465e-b8d2-8a81b80f3ded', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd59859c7-48b6-410d-b8a4-c13f7b6405ee', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7686251e-4777-4367-9f28-2f5f7631be35', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6cdbbdf0-0c57-4c6a-98ec-626dba53e968', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ac6f9b04-41bc-4305-9a76-0e0466f58650', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fa8b461e-fe40-45ec-b71b-4343f5722bc1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '58b619e0-e1bd-422c-b1b9-565fccc4c19e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b9ce2b34-5d52-4ddd-9097-873b467f360c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9d1d31d3-b4ab-41ef-9d29-060537cb1cac', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'df7b3720-5762-4c73-8e9b-d5a68f16f63e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9d7dde84-c1ac-4335-92a4-4565e89ae2b3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0e5aab48-9fbf-4db3-a757-8368df761f5b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '062c8f76-e463-4925-9be4-83bf4493d859', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c336124e-279f-4cc5-aed5-7af3aaec0433', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd2d09c80-40db-4434-b477-8fab63159a16', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '41cc8ab8-3729-43fb-9d5a-8a181b2f303c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1b7bf925-762b-4f75-9217-90d5d61903e3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9d05671f-b292-4aca-91e4-2a7001b357a1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0219fa4b-be24-4085-bad3-f8b5622c5b1a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd228e895-b72a-4735-9142-5d2d1a3be02e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ed71e82a-255c-4b32-9e17-f145525422bf', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'dd0790a9-5245-42b7-9b46-afa3e8229cad', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3368d1d5-2112-482e-a030-bacd6e58c7d8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Plants of Jefferson County Open'}, 'indexTerms': {}}\n", + "{'uuid': 'e64f755e-b1cd-4da1-8c6d-08f701bc8eca', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'cfb68912-93f8-42e8-ba71-24d118d6661a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ff3e8ee8-4e63-45ed-a55c-b0bd963a772e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1aebc559-c7d0-4c8e-a4e3-c4ef80cefbd9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '67777ad5-89e2-459d-bc0b-d27e148d2380', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '489c2d75-a42a-4819-b47c-1fd3c222cfe0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e85a0a67-8c50-4dfb-9156-6a8dc2497014', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '50a3b3b5-59c9-4d79-b695-fcb19ab52717', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ff0ecd5a-1507-456b-8250-670bc1228b8f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '12a9c207-f0ac-4da6-bb4c-6224a19003cd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4cfc65a3-a170-413b-973f-2ded87fe9d71', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4197c733-1be5-413b-866e-a4494d992ca1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8eb1a716-1f35-4b0b-b1da-8e4d617ecac2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '305d9fce-26e9-4992-92ca-5441b65d1a52', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a9f9bfcf-0009-4df3-baf3-5671c166bfd0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0802ac5d-5fcc-4d7f-bf89-c7fd809a79b0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0950ed11-c3ae-484d-b392-017eec08d0b1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8971e778-72d2-491a-8ab0-2093e6fb0f16', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '13a7075e-ff20-4a9e-831e-7bc95f35fe12', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7600d806-09e3-4c8a-862d-ef8cb8646a35', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4f2b291e-4201-4991-b11e-1b9f9ed39d8f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2ffcaed1-0785-4364-a257-85ed03e38cb4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '74854b33-d99a-4cd2-9f66-558d9b97135e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'dba1c40d-fa58-44a9-b461-9937a92fe3c6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9a7aa647-79cf-44e8-b446-ce480198611d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9c3b2cb1-c11e-4b45-80bd-25168b02c2dd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f28da9f8-9e85-4a31-b6dd-8ccd92a66505', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4b142e0d-2772-4f8a-b844-a8ebfd8cbb42', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b430b5fc-8cbb-46b8-a4f0-7a93b62f292e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f007403d-eea8-45b9-96e0-778c4a8eac39', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ad17f63e-4895-4003-8cd1-6f4c18093df0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '219897a0-2c6f-4a51-89da-834e34a41f31', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '91c0810c-f33a-4daf-ae02-233f893b2373', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6f3a4927-5cc4-4b1f-9226-f67affb4ff8b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e764f633-f4a1-4059-a0a6-3007157decb9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b4515d59-491f-4a0f-9200-792a539b612f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '026eacba-4478-4b7c-9027-ef7b3dacdac3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '09c1856f-bcf7-43d3-bb05-439a98a61f27', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fe0f3c70-422b-40ee-a549-1ae5ecc932b7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'cce95483-e220-401a-a4b4-c70f8154d1e8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'eadc50db-340a-4d89-99b8-dd3ff559b0d5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2aca9be3-276d-4c4a-b628-edf5c6b0bfcb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e43b99f5-ab2a-4854-ac32-01d7d9960514', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3b9554d8-ae0f-4535-9729-0c5e967f25ae', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '548ee675-935c-4dfb-ba2f-174c3bca5ea5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ad2cf002-2a52-4fe4-af4a-20a6089f2065', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f4ec6eb7-fd7b-4347-b758-2f044fc56b9b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '04dff58b-7f89-4909-aae3-b6985d1b93fd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6db90806-4c99-4627-8ee6-44a0ba26039d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '23d03bc6-ee10-4b8a-9620-7284726fefc7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f778cbdd-4d13-4618-a687-3ae0c7764df6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b7fc0b52-0ea2-4e58-bbd7-f3adac4fcddf', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '35407bc5-94d8-4951-9d7d-4975754d4212', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '34a52e5d-ef3e-487c-b68f-81ca7856f63b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1a2e3354-711b-45d2-887e-3abf8daacfef', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '90f3c7dc-1981-4c27-8362-7d12d7aeed30', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9c441a82-3a33-40eb-b4ba-362ef47438e9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c61e2405-fe78-4212-981c-e55c4b92de68', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3ff8688f-3d2d-432d-9bb2-37adf97be5eb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '122cb9b9-8077-4beb-89e2-8bc8cd44db4d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a762539b-bda9-484c-820c-1464bc99ba34', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b2e2a375-9fe2-468f-a393-63408c8bce09', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9864280d-bacd-4feb-8e03-bee3b71ceabf', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd635310a-06fc-413b-8910-94a78cdd8f65', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3e4686ab-ca67-4289-b091-eb05cea66e19', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1261f9ee-7179-4f2a-97ff-ce402ad15589', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6d06403e-4be3-44d1-bb76-5e81498b14b6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '46fdf591-6cc4-4190-9adc-1873d5d55f56', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '04ad33a2-281a-45f6-b07c-2514f8ea20e4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd3928f80-bd15-4bee-8492-54143bfb368a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '378f30cd-93bf-4ce0-abb7-00d774e7f2f2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9c285f97-6238-48d4-9c34-27477120900a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7ae530ab-efac-4b69-91cd-1a0eeb1b740a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ec0bfffb-a588-4288-b15a-e0ccc32bc89c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a6267462-bf41-4696-bc63-c1abb8756213', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '63f73e00-c997-4726-8231-d9b3aee14dce', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'eb51564b-1cf6-48af-9617-32ba83d8aa08', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '761de4ac-70a2-4f1b-93b4-e60705299f8c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5721ba4e-d6ae-42b6-9cfd-0ddfc24d150b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd7749815-2d9b-45bc-8f37-4a45e9953d0d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9352b551-57c2-4da3-8e94-2cc8db53380c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd02a1023-1f00-412a-bc13-8749e8783b93', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fea237c9-6f21-4ad7-a26f-6f728c584a1b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '49c2a039-04f5-457c-a479-d88ab6552848', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '05bf00ed-9ad5-46e6-87d3-649900c15557', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '838e8778-e83e-4267-9e08-8e91720f3ba7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a76e6057-7b1e-4023-9bb0-7c2c336594c4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '762b71e3-90ed-41e1-8bfe-b2b498f00979', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b1a1635b-7ab1-4859-9f3b-8e36f6bc7edc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '003f2d27-ee5b-407a-b24d-9fc2a69dfa66', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8dbc7f1b-3d0e-4aea-9314-f3d8697e2df4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fbc7a40f-dc1e-4658-8f79-06d74646f869', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7fe3aa78-f898-4286-bef3-82f77859b77b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9e8805c7-3b4f-4e08-b79d-9416eae61b31', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9f7bb958-6b9c-4b53-8ef1-0ae77f165a0e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5ead85b9-e165-459d-9abc-b2b15495f5ed', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7b22e21e-edf3-4462-ae34-977e629cc13e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3f31b9c8-adb0-4f57-ad00-6c1168d624b8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b0f4152e-46c6-4fcc-bc5a-7b3cfc403dc9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '13b959f0-9d01-40f7-b16a-a1309af07681', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5b1a86df-cea6-4ff3-a517-9cb03a6e0488', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7c4d4afc-baa9-43cd-9799-4fb4b3247181', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'db6a2cef-7f60-4c07-9860-90e7bd81ad32', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '81c78ae6-7657-4145-b219-60af1c07c952', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4299cbe0-1bcb-44c9-a73d-7c7ce4d8d2aa', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Plants of Jefferson County Open'}, 'indexTerms': {}}\n", + "{'uuid': 'aa84b0e7-2b24-4083-8319-1d77acced0b9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '43f74571-6033-4585-bd3c-1035d7986693', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8952582c-f051-4e42-9e1d-c0fd75538f49', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8374e19e-882d-440a-80a5-37b0b35c85d2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'cbc866c5-57a3-4dc7-be74-e4777be496fd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1adcfb60-81bc-4197-a2cc-9c80962d103a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5579669b-b785-4ac3-8a01-2c2896f17eaf', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'eb96f5cd-b91c-4ce0-beb5-bdc2d4babd79', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '407a1e35-9d1b-44a2-a258-f83e21e46bcf', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1b5d325d-0a36-4d09-b84a-bf0df6212d7d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '163156e2-2baf-4641-a6fb-23fe90e37388', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f1ac3f92-a7cf-401a-9a49-fd62c996e14d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '26285cd0-b33a-4443-84db-9eff1345201a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5d429e2f-44c7-4c93-9a5c-4a1d2d7d99ab', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ddfe3f14-ab00-4677-b492-762db9cb6a5f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5074e4e0-e2e4-4dfa-a096-ac00bd388384', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c921a5f5-1e33-41e1-aa5b-2fff810b2736', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2fd2049c-9d0d-4d0f-9b67-fb4b0684327b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '81071a0a-2670-4483-bf02-cf578a2f5693', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b0cc768e-204c-41b6-afe0-f54d95fd32e7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd8ce1976-2195-4d98-a713-0f31a1d42302', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b7db9124-371b-455a-a96c-68391d41369a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8984e7fe-2c76-4c11-8001-dacb3311763e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6935d682-79db-43a4-b6c5-15149521aa69', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'da4fe2bb-ba6e-4130-ac46-a998b9858925', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8ee0bad7-5a1d-4c9a-90e7-76f4ce294e9e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a26f284c-9cfb-4ef6-887e-c8510c366ea0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fce9e085-804e-459b-a66b-bb76838253f1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '468e88b1-447e-412e-b606-3a6f235a4bd9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0570e18d-0fb4-41cc-a267-27d36943c4f0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6b53dfa9-7112-45c8-83bf-408b194c0d92', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8f907f31-227c-49b5-81f6-7b81a9c78f34', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4aa63531-aba9-48e8-b7ce-ae89787af3a8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '678fa9ba-5d49-4824-9cfc-bfb42af090f6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '62b3b650-d037-4342-bb48-04a46dbd2aa4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '55f01c43-9899-48fe-a360-8d504cf36fb7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5b0254c8-97b2-48bf-9ff3-d4c17241cc30', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b152590d-3edc-44cc-9fa5-3ebd66177d04', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '78859fa7-4cc1-4fd1-812a-65515c917ac6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd0ae6348-b5ef-472f-8ef1-5085a2428c2a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e7c7fa08-a4de-4e2a-85de-5ba18eda4b84', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e8948644-4290-41a5-a817-04d5149a7b5b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7a8e725f-0045-4765-9ca6-b1f25e0216aa', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ec738414-8252-46dd-930d-b49668905b18', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3ddab179-0003-408f-9c9e-41923781332b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'efce9755-503a-43dd-9771-199af8541034', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '92e58295-0430-4a9d-8aaa-378d824c9f13', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6e6d2d5a-548f-4490-9a74-bc6e35c6a1be', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '47e287af-fdb7-4294-9e96-a6fa788b66fd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b1f0d279-cca8-451e-96ad-ac034e07876b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '259a347a-2ae1-4c09-b69f-9cf098debdd0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c43374b8-66db-469b-8652-48bf5ccb158c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4324e305-8df7-411b-a8db-024b390c11ec', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6b0b00cd-4c79-49a2-86ae-ecb4fe1b1831', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6e03f440-bba8-4d5b-9bb6-3aa64460a03d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd5256dfe-bfc2-42b2-a2ef-3a1604751a09', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd1fb985a-d576-415d-a900-1e28531ac8ea', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8b9496a0-4a9a-4dc5-b6c9-bf7c00549c06', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '06008588-8744-4839-89bc-49467d25cad4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4b280107-0693-493e-bcbc-4604be444b57', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '419d3979-2da4-4c3d-9fc8-0955cd73e2bf', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '41541114-cc09-4e1f-aba9-c22e9e53b6c3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '04ee40d3-dd80-4411-a759-0fd7bbf7a869', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e87e0bb5-3b52-4ba1-84cf-24ea753f4509', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6ad21cc9-5d7c-43b8-9cbf-1daf20159071', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6d7a93b8-30d5-4aaf-b8b5-c7f025bede09', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'af65d072-1065-4287-af19-67b3517c918d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3e41b5f0-bc56-49e8-9ae6-383f41897882', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f4d135f8-2a44-47a9-9e02-ee5d11f2991a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5b905643-cde3-4e70-883d-fd3e1ce418d4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '288dfba9-bee1-41f5-8bc1-08b84cc00380', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f72a8e66-debb-4aaa-a015-e87a8e6bb21a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e303b8a6-c07e-4fb9-8326-8b7da279c3a6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3fe4f4c9-8bee-4233-87d1-2cf28a2b0881', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ac7e25bb-ef84-479c-b651-207595ecfb0b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '604bf1f5-b81f-42d5-8206-ecd26e3757b6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '214d1e5b-e5b5-4d6e-aca7-1e24d00d7700', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '87061618-8a7d-42e2-bf1d-15b7cc2d1b27', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2e78bfd5-bf80-4905-ba44-60bec2c4e89a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2ec03486-ad6d-49d8-a556-54bec4c5643c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '89c5e433-9596-4bb1-a4c4-ea991e43c194', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e260fc15-44a3-4f52-8de5-4ff2584d3cfd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bf440eba-af19-46ef-ab16-19e6efd4dc90', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Native and Naturalized Flora of'}, 'indexTerms': {}}\n", + "{'uuid': '752c08dc-52f1-4bb7-8a25-05662a059d76', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd2e35fb4-2da6-4887-af81-a51a043e29eb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '38e5cdf2-9254-424e-9fa7-2763cc7e1d5c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c777b7e5-0f7a-44f8-9137-7223c1c5988d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4384844d-4cf1-4904-956d-d7c2d5fa1ac8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f6a11cbc-2750-4dd5-950a-75ec95df0d36', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5184216b-f349-4ef4-8b2a-13f9ba716a67', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f6b09993-673f-4e75-b24a-dcc75f9a20c4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8d352b6c-ba9f-42ff-8d6d-66ebc777ede4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4025affe-e345-450e-bba8-6781025679b0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'db75d4fd-6ae9-46e8-8d63-ce3f9e209e1d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '99e7498b-95bd-4f5e-a6c9-0316f459aa81', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ae14dfe1-3050-463b-8931-a07c0e21fe9b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2eb6be08-e524-4e2d-95dc-cadf90418a20', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4bceb979-d38b-45f3-9af9-58f5e3916f7d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7de19eca-01c9-4dee-8250-6c0fb08f0a92', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b3f8dca2-aae7-4a2a-bc02-5734a217d78e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e1df03ed-3c5f-4fc2-9b7e-951fa3aae860', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '97132996-3529-4f4a-896d-4ab8ae33ab27', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5cf31400-d7e6-4991-9fbe-0b888347ba8d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '00b50008-460d-4533-adb0-2a403be9f5c1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '576a7136-cf14-4efc-8e17-af2e93b4daa6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6dc9d72d-63d5-4335-a272-94f3511f3a91', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '09882b9d-1dd0-4d2f-8d86-160c6184af3d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5048c6ee-d804-4067-a8fd-137a7170ae35', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '28961370-c23b-4c0e-882a-88ce3919de21', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '35a26377-3318-4962-a57e-5208400d8167', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '03fa3536-99eb-4a9d-882d-de81a46ee14e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4477be4e-caa8-4536-bf2f-fb0de359da11', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e635b2a8-075c-48bf-86ce-018f40e515e8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '320aca8a-c912-40a7-8666-d4658248aab6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ca9a1e4f-e5ba-4e8c-9caf-5cb4875a981e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '96f0d258-9a62-49ac-8dda-d03b912a6312', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b70a09a6-92ff-48fa-9b90-5ce58d41f2f9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3e0fed54-f915-4aaf-bd92-cdf49552ddda', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bb2f83ac-c5d1-4940-bc44-632ecf7b9c69', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '90a5db32-2e52-456e-bf79-0969c01af2ba', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '19143c5a-c6f9-423e-b75f-3ee01647271d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9085d26f-2c32-402a-b131-5ac7509d909b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '33f92699-a93d-490c-b9a5-453fbfc849c5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '716de181-5bf7-4a14-b681-ed02cfc66fb6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'cc4e4462-6c92-4159-9a92-a04efab2225e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6797c140-4da8-4a7b-9117-2698bf01fc88', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b5dbd0a1-bfd8-4b3e-92f8-30043a86e4af', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1cff9e12-8302-4ad1-9d31-37c4b63a98aa', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e38165c2-c7a4-43c8-8824-57d27a788805', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '66511930-5dd0-40c1-bdb7-6bc3c8101fb7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e3e685bd-97a8-4ec8-9611-ef7b3733080f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '41d77f91-9b6f-433a-ae44-736d7438bdd2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8cee653b-ac2f-4952-b626-827b74b0758f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8b729483-9ced-45ab-971e-254fcf5b854f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8e73cc4f-1926-4d47-a82b-2124925b63d6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2ec31827-4280-443c-b5e9-7f38e44a6393', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f4a5cbba-272a-40d4-88db-39c30555d0a7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3f2d4ccd-77f4-4a6a-a3e7-da3f823eee15', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ccfbcf41-ef84-44f2-818f-cb3369bfc9bb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd1bbaebc-d092-49e5-9da5-2c8ef756aa53', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a0adc87e-c1ab-4bf4-bef7-09c3938f7cf8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '50fd4114-38ec-4506-a650-87382748660e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a5f89988-e519-45fb-a3f5-fb8c5adcf9f3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9419b5d0-9341-44c4-b517-60d05937487c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '762dd059-e818-4c18-9652-596819ef18f9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '63499961-3572-4dbb-92f7-fd6222ff0de9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '52b3513d-4560-4289-aa30-535395eb5e15', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '751f598e-4f72-4d21-a5d2-6e06e5b54a5a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c7b1001d-3de5-4b65-b8f0-9e4d833972a6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0da9989d-4828-4176-b100-78601553098a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f872fa6a-8e56-48d3-9e71-d9c2cdb3f8b8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a6128f9d-9390-4b60-95c7-b8a2acd5a45a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0a5161b8-3800-4432-9f21-8f825c6f3b3a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '02002d85-c145-4691-a487-8f547fa944c8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4606120c-efb6-4d5f-a8f6-d1a1cc5d5807', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '52b28896-7204-46ff-99c6-63549941f3e7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd9063bdc-4bc2-495a-bd8c-c1344910f1a5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e04c5b69-6933-40f1-84bc-39f834773960', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ec6129a1-6e59-415f-9849-f96a763adba4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '06d39005-1967-4db0-b128-6efb018db131', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'af3b38dc-37ce-4dce-beb3-d07af18e7db1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '992438ac-5b65-4c4c-ad73-d621131b5f45', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ff44c271-e399-4f40-a0b9-0871e41f0543', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd70bcddc-d903-434b-b827-05b02054d1c2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '848d9805-b98b-4ec9-aacf-0b70dcac9eb5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '275e1f93-01cd-4765-8921-37a3d575ea8d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ee40c6f5-dc21-4f19-a817-c5a78f40d6ea', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4c9c357f-c688-4d3c-a1ac-192da6e92e44', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '36d51d0b-c8c4-4683-9436-c4ee45512796', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c94ba176-a4d2-457f-a715-b5426b3fda24', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '259c7afb-9582-48bf-bf0b-b5fd18f5a3e5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9a127284-d6b8-4c7c-bef7-541079d39491', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '96b23f78-08de-44cd-a4dc-feb6521a9268', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e4d73d20-14b3-4b15-8bb1-74f9d83e3b58', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '803f3be7-ae92-4f25-a06d-c473c017911f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '18a4a160-7dbb-4cd0-9acd-f8b1cfe956e1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f337584f-3346-4cdc-9d2a-0aed7c338092', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '084741de-be5f-4e53-a50a-7d07e620d8d7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '009bd53c-af74-4e11-8b4d-19d1b93666b3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '27c681bf-090e-454f-b70b-daa3321b0bd4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'aa5b8998-cf89-4cc1-b87e-2e1f3f173c74', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '047db3ac-998b-4d80-917e-53496a1840c4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0be2d437-8d34-4478-8bd3-75adfaa779ee', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2d78aa5b-dfaf-4de6-a9f5-4c05a37c65df', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1b696b0d-2557-453a-b929-57c68dc81a7f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ec64fbd6-3ab6-4605-97b4-469cb643d9ca', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Native and Naturalized Flora of'}, 'indexTerms': {}}\n", + "{'uuid': '27eb8888-0d51-46df-9569-1ab0d15ae0c7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b88a16fe-6122-4908-a207-802cf3003262', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a09a581f-04d3-4da7-9116-e1c738756e80', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '898ec1b5-dffc-4b85-94d4-95538e6ac3d0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0b3b7303-03ae-4b62-b5bb-566cae2de383', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '86bc3f9b-9f15-4914-b191-b5fc5dfaeef8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8c0df66b-177f-4c9e-bd47-5564462ffa73', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '700bc496-e707-484f-9187-6e55c341df07', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7aed2f4a-760a-49a3-86b6-ab791c913c82', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2137d07d-d63e-4c1f-afd6-807189ce658c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6799d0d3-7870-496d-a0d6-0a25bba20119', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '155ae98b-c647-47c7-8e35-bdc648a9192c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '710612b7-7073-4368-99cd-7d302f79b79e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f48ed898-cecd-42a3-b5de-99cf3e9cbb0f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '12933598-068c-45c1-a003-baf8466660f1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4a22b608-94cf-4c87-b089-b47a5bbfe358', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '287af94d-2668-4534-919e-49bce1d2d07b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6158c857-b21c-4bad-8ad8-2d9fffac0f55', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '97874647-6e2c-4259-b2e3-67b5f4f08b9d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0b2e1208-ee19-48fd-ad99-563036984abd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '76c4de11-697d-4909-b8dc-a479b0740656', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b4e26984-6df9-4e08-a364-efa3dd09d647', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '561a6021-acad-4a23-b196-c740dcdac43d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '28400ade-8e03-4dbf-844e-8d5fea068474', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '81dba8a9-356e-4651-b5c2-622236c6a9fb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '63136ad5-6f58-4e45-8c4f-2d5bd1b83ce5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6220d90a-2fcd-421c-ae5c-4a0b2dc6c94c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9f91156b-167b-4932-8dd2-b7f2a4d90ad3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e2d39e31-e65e-49a0-bd67-fee3c1a21bec', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bd613baa-6d60-4bfe-b9b7-f708f2fdb369', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4091ff0f-3440-4887-9eb4-de61d3f05bde', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '980ff461-566e-4406-b88a-ca34d8e89690', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '5140c272-732b-467e-89b3-015a9e0700fb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b7eb8595-90e1-4a8a-baca-183e7acdfc53', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Plants of Jefferson County Open'}, 'indexTerms': {}}\n", + "{'uuid': 'ff2ecff1-bd68-476d-9ea5-671852b22c83', 'type': 'records', 'data': {'dwc:basisOfRecord': 'FieldBook'}, 'indexTerms': {}}\n", + "{'uuid': '15bda212-a4a1-4312-85ad-4f91ed6e1bdf', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Photograph'}, 'indexTerms': {}}\n", + "{'uuid': '7c577caa-86c0-413e-bfc6-1bc0fc17d6b5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ad0a3266-98f0-4e6b-9f3c-4b62a99eaf87', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '05c7001a-8822-4b1f-ac4a-3b33d1eebe40', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '11077bbc-b4af-47c1-826a-c132a10f17eb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '65dc3e7a-4ce1-4c18-875f-f3a024b7b879', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '45df3ebb-6974-4486-983e-6bb7c2ca4333', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8d6c0c94-4886-434b-8a2a-8d5f4645757c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd736505e-91f5-42d5-8bb0-8eb38ba93eb7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '41494c93-3517-43b9-b230-8a19981c16f2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3a4c9c5f-2a16-4a82-a713-ac42bb708abb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a6ed17c7-058e-471a-89b1-8e80298e11b8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '13cc03d2-3e70-44ac-92f4-b11870d80195', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7541934b-235b-4de2-9a09-1420b3a02a84', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3eeb3fdd-f308-4397-ab12-7cabe273098f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'dec71d82-ee03-4222-ab8b-1cb7fd98cc54', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '99d48f96-db10-406b-b61d-4e1c387ccd1f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b982352a-7a7b-445d-b43c-a847b191e748', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e5fa899e-ded6-49df-bd45-cecbb5119577', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6c2db89a-299d-4a24-a067-36fe7e9653fd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '68e3efe7-d6ed-43e3-b5a6-1f357b92f7f0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '04fef646-0e82-4d7c-b21a-fc9d86bf93f1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fb264303-516f-4be3-89d0-fe4fef250ef8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '646dbbf9-deef-42b6-a76f-9937dff28e26', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e2ffab88-345c-4786-a726-8dc21c3d0098', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '23060aae-697b-46cf-a504-428b1d4e3160', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '454b95ef-6f38-41e0-84a5-c2cbd2908a1e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2e045af2-2e53-41ff-93de-4d76a2b1caba', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5dc3c760-6f11-46f1-ad6b-b1913c7ee5de', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '038ab183-903b-4185-a1dc-842c1ebdc265', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '232f5969-3f94-4d32-a946-d0bb37e1520a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '55c70fdf-0e3a-4bbc-a16c-fcbcbf9f74f2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '21a296cf-198d-41ab-8f4e-238fc3cf05b9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'cc9c84b5-b391-43cc-9b6b-cabe6ca8a690', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ddc29eb0-956c-460f-8217-709570bd3612', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6821f643-7718-4c83-9afc-b5855d8d760f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e0daabfb-f69e-4944-b4b6-b2fd3d6c1b20', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2d4d3995-7fb2-4fa3-8616-66c3a5c558ad', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e55ae1dd-026c-4514-bba8-08ad1d00357a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3af4103e-53f1-4306-8fd2-a764c1d21439', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd425d7cf-31fb-45bf-8bc3-c007c2b3d53a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '79e8e84a-3cd2-4b0b-a2cb-8dad25927864', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'aa3f7394-0e52-4e52-bb56-6b61ee72fe45', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '08b4d2b6-259f-42a4-b0f8-071553b2b746', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '43615248-b6d4-428b-b0fa-00fb9a652ed6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9e37d976-8fe6-4230-aaf8-1fd959531741', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6016bc64-8e56-465f-be70-204c42ee36ba', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ac73d39a-b2e5-4e0e-92d9-46160d36896d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fef09769-ea9e-4c0d-a31f-60555b225f0c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e358d24b-be8f-4803-9e3a-ddec55ae8875', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '07370102-aa58-4542-beda-8bc245ffdf27', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1e11381b-c421-46e2-8ab6-b2f204586c15', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '427a528c-7270-449b-b845-4ff072c6f86d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7f94641f-d172-45f9-a794-620e6466e85b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '699c4c91-d70d-4161-9d8c-00c833ace714', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '76c6ea4c-ca68-46da-a364-f1841c595232', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '124ddceb-c6ca-49d7-8038-30cdcdc7ed6d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '15b3dd9c-8520-4b63-a373-dd37ba16b951', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '945ea845-341b-4467-bc17-bb8aee4022e7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b2ce6fef-a46e-43c2-9ab7-9ac6946acdfa', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '99a8ff53-db7f-4eda-ab0a-19688d64cad9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fe3c814c-0b20-4075-b202-0ace767e383a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0d4edb55-6814-49cb-8218-460b88ec2854', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '291661ac-081a-44da-abeb-56bf0907cf9c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5622e883-e475-45fe-832a-1cd1a96ff62f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8fb632ba-5524-42a6-9165-1c44bcec5752', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '88ba4f3f-004a-40bc-9af8-b5399ef7eeaf', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6401188a-fed5-4bd5-a97f-afe1af35fb9e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6beca7f6-932a-4632-a5d6-3b2300f487d1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1efeab1a-1b5e-4dfa-b31a-dd08e4b79760', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c48cf610-fad0-4d51-b178-50b39430e1c1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'beea8ef7-a8f0-4569-93be-15dc6c2982cb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c25acc1a-a56b-4827-a544-d48594caee81', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b634570a-41d1-4d4a-a6b4-8973e5619759', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '13fa83a8-9bb7-4f74-b611-e03a7aa20cb1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c5376cfe-0c5c-421e-a9a3-711ed24fa384', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f87bf3ea-e956-4410-bb27-ce8146df6107', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2a9eb27b-7d48-4301-863f-7e739cc839ba', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '267a0ed0-af1c-4cd4-8bfd-9dcfa5116a0f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3bb260d5-ebe6-41ce-94e3-68fb4edb3f59', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'dba9b3e5-2923-4834-a303-58c6d3766a95', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5a01aa2e-4a45-412e-add3-0aad2a6644fd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '470d4085-2647-4ada-b1ee-e8f8b4fb2d12', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1e263288-c808-4163-932f-2bf7c8218f2d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6e294a9c-84c3-4a6a-882c-ba1b1cff72bc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b426ffc4-c627-4469-ab5b-a162033d0300', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7df22471-caba-41d6-914b-3db1452f09a4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '99145d95-c87a-45af-b246-be74b70d641c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8ee849d3-ab94-446f-9d54-591f50963f83', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e2b411bb-4bd2-4240-bb05-87bedd7da887', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2081e7c5-64b3-41b6-ada1-ff3dc8bd3e09', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0f368ab6-031c-4b9a-bb5e-9620aad871da', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f7cc624a-311a-42b4-98f7-cdca5b977802', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e52360b0-653e-46d8-a814-f280ed394aea', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9363502e-849e-480c-a173-f4b5107b75c2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f0dbb6ba-d865-4c49-a51c-01414aca26a6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '125b4df4-7643-4e78-95a9-7ad2a15376c0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '26483ad7-f18b-492b-a3bf-a3547e688c7f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ce3fcefb-25b3-47ea-8ff9-a7e04867ad2c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0ec68aa2-44db-40c6-97e1-b019de01d2b1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '97aed904-3454-43c5-80ff-75abf3bc5c54', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1dc4a172-931f-4489-a095-5f28198b2dcb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3680f96c-cc92-450a-84ae-0d50a4e0d387', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9d879ebb-1958-445d-91c5-54ed624857dc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '02ce3d90-16df-47d4-8c67-c604aeba5154', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3f27c567-b62a-4154-8d37-0ce27e1a624d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c0b3a6dc-7d1e-4b65-847a-787adb69b555', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '71357189-682c-4362-827b-9478cf520d1d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5da7766b-a80d-480e-b7db-b8c2080c4cc9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'df371fac-df1e-4aac-b668-8c91df3198e3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7ae0e667-8aca-4a0a-b1ea-0544c97b1ad7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f73619ee-2281-4d28-b40d-4763afcdcf4e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a30b0878-3737-408f-9057-7b23ca07fcb7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ee25c0f7-c1d7-4291-a988-02a56a645de3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'materialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4e7e6f71-d693-4396-b22f-218ff34eee22', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c3454c1c-abba-45ec-b7f5-9c965bc0b8d0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f6873391-c2aa-4ea2-a460-5e16c88b0332', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '07f19cc6-1a08-44a3-887f-d0ac492d0d12', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd04827ee-f1b8-4579-afad-c8103eb2b8be', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'eade69e7-2b95-4281-9514-41b18b7d9fe2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1a395053-f6f2-46f5-943b-a62c34b5d625', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '473a97d5-985c-4354-8fe2-92127d5fc5f6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f059bc23-ef71-4a8e-a38f-bb5240159547', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '14191a71-fdcb-4e23-a03e-975bdc4676e3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '20efcd47-7b34-48f1-b853-a42f5c14e781', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd793b356-ff1d-4fda-a58d-60f4f026f9f0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '689f395e-a532-426d-9201-9fb4a8aa68a3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'dee7f1f8-76fd-4752-99e3-0781df096f4e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9d1979d2-14c8-48fe-86bc-9fb37b2f8b16', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd093c46c-78bd-451d-8028-f91d84a7eefe', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0e3addda-309b-441b-bfbd-a3e4d3501044', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c92424de-c80c-4543-8ef4-1234b8d59293', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2efbcb3a-5044-4469-b010-a8cb7881d986', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1deee18a-f016-467b-b864-51aeac60e9ce', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '462dc86f-32ac-499b-8e92-a0b32386aa14', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'aa30a766-9a17-4426-a943-9cee92a418d2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3d6642d5-8e78-4405-aefb-96a6a3757332', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '44cd7a76-b10a-43c1-8948-552531f04df2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5e4b5fc6-47bf-43aa-be43-54e263380e46', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '657ae314-17ef-49b3-ab3c-8cc2f3424a54', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '36e52362-4e3b-4b92-8821-c82b93d5183c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a417de4a-2628-484d-9466-3270c222285c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c56030f6-c92e-4446-837a-8daae25eaa1d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9fb9c0ff-1ce2-4acc-a152-74443d7d290a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '42ababbc-72cc-4f5e-88d5-a95dd1c66c1d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '91c36222-95f3-4802-8bc9-0d54db3e5367', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '44940242-fea9-4d84-9224-1239719b3e36', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '171ce055-5908-4fb4-804f-5a57abd82a52', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6790d266-cb91-490f-887b-cf41bc49c453', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '50570a90-cec6-43e3-9932-e1e22c5a4fd3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8229c0e7-f602-4570-a70a-859986886410', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9cbfc643-5d80-4757-9ab7-9e6dd71f48b5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fd6115d6-d838-4b83-84cd-2055d0e606cd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'aaaee5cc-8ed0-4482-a1b8-b65c817ae69e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f24ee446-f2fd-4cd0-8dd6-999d9eb1e736', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3af363a3-8cf7-44cb-884c-24c7057a9bb4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd55cfcad-bc50-4f73-a2e1-c8c01f635670', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3d7e4837-3fbd-495f-86c9-816eb5506735', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '050a56c9-9129-407e-b3a4-b256beedb90b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '056d6209-2eac-4aba-af0b-fcb6c90cc2e5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6e1f806c-d720-4962-99ce-9dc0d27d9514', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9f3352bc-ebe1-4c1f-9c7b-f06f3a95b797', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '48c08f1c-ee6c-4cc2-9a9a-33b6cae67399', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6bb9bd31-ad11-43b3-931e-a885440ed764', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b597bb29-2b12-4f5b-a223-7f0a7bba08cc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Plants of Jefferson County Open'}, 'indexTerms': {}}\n", + "{'uuid': 'a7646d5d-6c2b-4852-ad47-be972c2cd1f8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e9c9811a-3f37-4c51-b575-2a897fb91c4e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '44e3b2b6-c347-4874-ace6-f3311c36bcde', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '403df249-74af-461e-abf1-0ec43c2dcb49', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'aa3821a3-ae02-4a55-8151-66084cfd6791', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e64c0028-4024-4588-a94b-148480a37160', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'cc8b79ea-122b-4558-a366-ff69b5c7d7c2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4b4d73dd-9118-4d89-ac77-70ecb1192618', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'aa990eeb-68bb-4679-b235-7f33caeccda6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '37bbba70-bb9e-41a3-958a-cfcc940ed50c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '69337f4d-a345-45b4-9f4a-8243346dd416', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '30e7d708-ce2c-4445-892d-8bbdc0d97899', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd0e1466a-7678-4947-9c5e-4b2e721ea54d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ad178b8b-9bfe-455f-b797-2b19177a9623', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3ff8330c-b0ea-4544-b709-c36d67e3d195', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'dbe41156-1a2c-4894-94ce-704146a94cf8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '432621ad-962b-48d1-a5ba-4067575fd525', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'aefaed2f-c22e-48b1-be53-576f1c6bb806', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0f6e7cf6-30e6-4057-b8a4-777d2d748b30', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '781da2ca-528a-49fb-825d-72b0f73e1dd3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd4b2b321-968d-4582-9ada-3aa4efd5aa17', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '12586018-d468-4085-89f7-275d0295cc26', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '44665fc9-90d9-4727-aaeb-6ae20c2cabda', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c18ff875-d01d-4469-886a-b8ff3b976ef1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ef951a70-68b6-4fbc-9391-0f3b532a55f9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fa02b909-385d-40c3-961b-ef8368cde5c0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ec234d25-aa69-4aaf-bad8-59c7cd535034', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '163e678a-a402-4e01-8516-82e17f7ffb0a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1d32ac0e-2e9c-47a6-b3be-2155f737f0b5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2142c219-77e5-4a20-8347-2d50258ac5fb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '08ab96e5-27e2-478b-b230-5298a4393232', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'cf89814e-71e2-4bfb-b1bf-5a6a545f064a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1ab97f26-4bb3-447e-9eb4-5af7218a95c1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2b8d45c6-d3df-452f-9498-4d7fd715cc4e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6be9a3ef-c085-451a-8a0b-b48d17b81153', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bc413e27-e960-4a3a-b1f7-a4f26a7aea11', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'efba12dd-7d45-4be6-b033-bfd41a1a29cd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'dfa56564-f881-440c-8e87-1ea0d301f178', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f4e0dbfe-1c91-431b-bac9-e70d72b7c573', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ffe34cfc-8ee2-49a7-acf7-7fc23a7e8209', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b9ad29f9-7b3e-4cc6-9e83-be6f7e1f7ac3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '043ed3d7-10b6-4bd8-a133-63cfac64682a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '15c4edc3-544b-45df-af3f-2b01ca6cb72e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a1f891c3-c5ea-42dc-b3d0-765c21dfe808', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1b6bf59a-a54f-44e6-b176-9f7cdd841781', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6865a3ec-5c08-497d-b9bb-b918ef0948ab', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4b8ff76f-7fab-4528-8785-1a9ded6e4ed2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '652e23bb-0e8c-4eec-baea-20fe107c8be4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c7c6352d-2841-489d-91c7-c980fd9ac98a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fef6b587-9b73-4be5-bb72-9beae2d8dc6f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ab28cc6e-2ca4-4946-9340-c1291faa7c69', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1d4081a8-5bd1-43ad-bff5-31524dda32ba', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5e5281df-1101-4750-b618-0aa81f868be7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '21e6cfe0-cb5d-4a1c-9d0e-7a5b22da3030', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3e621045-579a-43f9-81a8-3da1b8dd889d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'afbb0137-565a-4b56-8302-d99e8b9929d2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '001a996d-09ce-4be3-9a8f-d8747ea8a215', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b5eb4df5-6863-419a-b7ad-c590a1935c1e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0e2079f1-b955-4300-97ae-e1cf8a7d0b42', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '911f1d31-2ac8-4eff-a7d2-430701431e8e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3fcba71e-8a0e-47bd-8e18-c79a0977558b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e443b865-2686-466c-9627-6cab7d0fd3fc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '90b89e02-432b-469c-92b2-8bdb357e066d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6fc4b14d-ed8b-4e64-85f0-fa48d6bd0c07', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bdce40b7-2794-4f54-b668-2c22659acf15', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c3fb4097-e276-43f3-b38c-46701ff487dc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5e2f6fe3-ed15-426b-9842-80cc7cd1386a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fee3fcfb-ab02-4596-a547-4f03de9ba7f8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1167b966-8153-4e6d-897d-91557bd7f118', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '53ce3f56-4d73-4ed1-a675-bacb4a065c4c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ce92a02b-6b3b-4da4-99c0-f409cd30653f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '41f4eb6c-bf7e-4d33-90fd-c10091b62325', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e87ce094-e502-4e9e-862a-f0a77dfd4ea7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ec2760f4-34ae-4b14-9394-bddac2ed67e2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8d33aef3-ff71-448b-88f7-a8e08210329d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '43538a45-3a0a-4f0f-9fcd-bf8cbcc6bd7d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8124ed6b-3d70-4422-b9c4-e2f13e052b7f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '95f432af-e2ff-4baa-b165-837be07b001e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7707f4c7-9e7d-4601-9999-5170d88357f8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7821b39a-be26-445b-b828-b3795f17bbcc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '58f1a817-4bb5-4801-874e-e614051581ea', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0d5a6aca-a216-461d-a2de-7411e360919f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c23603bc-06b9-4907-91f6-6f61c2f0a12b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7d0a9fe3-55a3-462b-b6ae-f66999ca5a54', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bd177fea-4d20-4136-ba0a-7e158984f249', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '82e82e02-f8d9-4862-b45a-40ef37c3f774', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b8167fdb-8561-4b68-b715-d0be8b774775', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7a0ad6d6-4baa-4c2a-a610-0b69749fbb08', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6addbe04-3b33-4f77-bca9-696565ac2858', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9c750b00-c75a-4319-8cd0-950b2a871387', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a9c9cd02-f9c0-4130-8773-c06831258710', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'cec5d33f-590c-4ae4-9c63-1bb7aef6c1f9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '053c613c-868e-4261-a0eb-55ce31b3786c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'fe625fb1-01c9-432c-b2f2-407fb1a83fa5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a5ccbd62-7084-4de8-aa8f-79e9ef9c47ff', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Photograph'}, 'indexTerms': {}}\n", + "{'uuid': 'e2213271-c6be-495c-9fd9-d6748a2205a5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'aa79d0ed-e245-4d75-8f21-41b42bd9f0b8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c8b94372-9909-4954-afdb-4394e5654954', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4c71e7de-6edb-4f53-8487-42364d1ad859', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3e3c5bc1-323e-4216-b824-74a6bcf46585', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '948aa838-7053-4694-9adb-4b59417b9ada', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8e9aa722-be2d-42a9-9a71-84fbb6f5b7f0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0fe593c7-e0b8-450c-800a-34ae17a3ff22', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '145676d4-af23-4a63-9655-a064464b65db', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '66445d4f-1bc6-4157-b479-5fbb57b99f48', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd6383606-c972-48cb-9000-002fec88c488', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9bae8dc2-cd83-4961-860f-75fc2d4ec0b6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '15915b92-2ebf-485e-a4c3-b1b559d0c325', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '692ee61d-9160-4455-891a-c7cadbf7d430', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b78125f0-a773-4dc0-a521-e0132ccdade2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c47707df-61d2-4c34-9095-df536c41c4d9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '23b35ffa-0fea-46e9-aaf5-9577c5e97e05', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ada77c97-38d4-4b8f-a06a-2a8ae90137a7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b2418936-e967-4a20-8c9b-e1593093924d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a5732b39-8287-4ab7-a5d7-87f37af1d610', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bdb5e6f9-8c0b-4570-bc85-50f73c244628', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a0c13b54-c2d6-41ce-9bfb-e942750f8c10', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '677fd313-556b-43e4-8bbc-7b06882571d1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7fac35f5-7d26-4c46-8aa9-24e388ea327e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7ddf9355-666b-4505-a8a9-bae002c91633', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '06bcbd95-f7e1-417e-9f6f-d3a611cf4bea', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c1ee45dc-0dbf-46a3-a4d5-2add0a8430bd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2964608f-775a-4640-8a43-5a4a9e0a7286', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '31c92b2a-ece8-4f6b-b37b-ccd3fdbd7600', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9beefa6d-9d03-487d-9b1e-4927d4847ce9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '563d937d-bf58-4060-8475-a19ccfd04cd9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7a2278a4-fbac-42e4-916a-4070a0986bd3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '070869ec-1ca2-48ac-9e59-8b18b295cc18', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4ab8dea1-fb62-4a54-bba0-2276d48c3b03', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ae292afc-ff55-400b-819d-7bf7615c1312', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c7b1ed5d-79f1-4f3c-a04b-4705bd23fd30', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0b802ca4-78e2-4507-ac10-f123d2376169', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6f403b66-0ae0-40ba-878e-80be07fcfa76', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '27b7153e-86d1-4b13-bbb4-40a5d62565fd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bf25f7d8-63c2-4665-834d-0e0d4c557a41', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0442068e-d03b-4b66-addd-a19b8d7be3a4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e96c64f1-9ccd-4510-857d-6d61312080b5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4ecfe9d9-5d49-47b0-9b56-fd56ec524034', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '940ed82a-1190-47d4-9f87-b9a1d54b687f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8fd86735-232b-4ef5-bb5e-a27f663c1044', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3b19ff36-3520-47ec-b6b5-7f8bab5bf65c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '5bbaeee6-2143-4bce-ac0d-53f884183cdb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0259f110-ce33-40e7-aa6c-f4aea552aa5f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '51eb7002-43a5-4c38-8292-1af93f924b73', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b052e07f-4f63-4bab-b3fe-077ac7523903', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '46da55b9-b701-4833-98fc-ea3b6b9c490e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '42069a6a-bcd4-4db4-b819-14c8fbc869d3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'dd6c3d28-1bee-4c7a-8e6e-0bf128999019', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '99df2199-e9be-4f0b-813f-6a1d21fc0396', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '32024e0b-dc82-46e5-a5a5-3b34d3e594dd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '629c2cc1-ecb3-4ce8-a3b1-86ef201bc9a4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '329a2842-61d1-4eae-b958-2139e781d581', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a026fb2d-8169-49ce-b3be-14bc07eb2b09', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3767fbc3-4c77-4e3a-b3c5-e0cc2b58806e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e039e019-67b4-4e48-b460-69530678b55c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '56477c59-1e1d-45ea-8fee-d20a542ea58e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd9a71f3d-bd7c-47b1-a74e-ee1239b2f5ea', 'type': 'records', 'data': {'dwc:basisOfRecord': 'materialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3115c8c5-8e68-4e2f-9fd1-5fd023864662', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '203b8ef2-1ac9-4c1a-bcdb-320a86bfae0f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4595db70-f9fa-4d27-b553-9197d194cb79', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '734f0b3a-3640-445f-94a1-0f5ceecf54d9', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1c99be33-8f8a-4fcf-af2a-578677260d7c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd7796f6a-7be5-468b-b039-6d558ee62243', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '44a4cee6-2118-4da8-a34a-e580dd7afbc2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd06b2217-0292-49f9-8e59-2db5c3e0a999', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a1930702-0b57-4ea1-8402-e92ef5d8bf4b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e95bf27a-0ede-4732-902c-4a2c319721ca', 'type': 'records', 'data': {'dwc:basisOfRecord': 'materialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0483cc11-dff6-49da-a30d-ac3919315ff7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '40fac5e5-fcf6-44d5-8890-dc10fd4360e1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e96713bd-0434-446b-b20e-046475e0d6fe', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'adb5994d-5a36-4ac2-ae0b-fa7a0245dfbc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '83d28915-0de6-40c1-ac42-1e0a8f344fe5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'caa0e5d1-dec0-424a-81d1-a54771931ff6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8917b98e-3433-457a-8f17-73649c68b112', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e4a10f69-9acb-400f-91cb-fe0ed8098046', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a30d7838-4626-4be3-ba90-e6cf9f6797e3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7e7356a2-1970-404a-a6e9-c56af2264bc3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': 'fdbfeefd-5b6f-4d42-a953-a8ccd814ddb3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8d3b3921-5494-4edc-8eab-9b7865b62f77', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0c42609a-73a4-4870-832a-27c3c22fef7c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8511710b-a59d-45d0-abba-f48f923a5138', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b56cbae8-8304-4cda-a0e5-a9120fdf3810', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bd0c9cb4-7370-46de-82d8-f10cf31c6a0e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '02ef0058-a9c3-4b6d-a5c9-326f72a5206a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6547927c-5c76-4cc8-a588-9a606b0b30ec', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'db58f770-2be3-4dc4-9d8d-9cecc8ce4329', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c49305fb-3ca7-4597-99a2-4517f33530d0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': 'e5fe8257-6ed3-4bf4-94a3-d5e0419046b6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '8ef30681-eada-4216-b6e0-b5057d311868', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c2485312-4a27-42bd-9d25-d4b99be49949', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1b746cf8-37f8-4596-b9cc-993f813cf8a3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'af343385-dfb6-441c-97b1-73ce4b534977', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '798855cf-05ef-45d0-9f80-ac3cc6d08e3f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8a4392cb-2a88-4083-9117-8953f2d0615d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f60d3680-a27d-4052-bb72-ab66110491c6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '95ded52d-394b-467f-a6b3-0b0ad8e53048', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8386c9b8-a7ff-444a-8e6a-27dc9da263a1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ac78511d-ba92-42b6-b81e-144d5c55bb15', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '248fe70b-8e1c-4e6f-9fa5-73bc1c25508a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'materialSample'}, 'indexTerms': {}}\n", + "{'uuid': '138ae61c-7b60-49f4-a072-38548239752d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4d0d730f-6041-47bc-b58f-30e3b7f75146', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2dcea38b-5fb4-4f59-84b5-01050ba8c02d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b134c8ec-40f3-4cdb-8d17-a94e86074635', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3d8ba431-e4ef-49bd-9277-02be96c45607', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd61db9a3-042a-4d6a-8bc2-3417455d1aa3', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'cbbe3dca-3a9e-42e0-9906-5e444ac565c1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '6a7b5cd7-1a01-40f6-b454-8baddc3b5430', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '0fd189e2-db7b-47df-b414-5dd719a9ec0e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '371d6ae2-065f-4156-8240-b5f7d51ee15d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '310a3d4c-15b8-43ea-bed6-1e0449c778e0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e8e91ad0-0c3d-4292-8dd5-34365324da71', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '06cd7078-3b10-4f28-bfb8-b5a72a5c30f5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '54db79dc-6b2e-49d4-8e05-9d3cacc6e496', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '655df0f9-1cc4-46a2-82b1-d9133e4df86b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '03d9f15e-ca14-4d18-b7d6-19592c3928b4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '0301f119-2218-44c2-8893-9c0c63dcea9c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd78225f1-1b9b-4101-b2ae-b34dfb948832', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e8797d7d-5403-45a8-b747-619ab964568b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3a679384-c581-4c13-a264-71701b6b2bfe', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4a2dae76-dcb7-4964-b9ec-81b5ac56dd67', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e494699e-c947-462b-8563-b747333c057b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bd28188f-68f4-4a62-a634-ab8c33c5368d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '98178e76-5fdc-4023-8b45-75ab5fed0894', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'b5578714-6db1-45e0-8307-9d3c78e1a1a7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9063c3c5-976e-4709-9ed8-a56e5ef6794e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '36509c5b-5d5e-4ec7-b2eb-b550465145c2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '837a550c-d002-4094-b2aa-7e3fda56494d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6e52d11e-1556-475d-adbb-a58616ec512b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bec98eeb-bcf6-41ee-b29a-652977f58bb7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': 'b70c4091-2b49-44ca-8625-77355efa3acd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd735e090-9e80-439c-b1db-97450edb0474', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '926988c5-9ef3-4510-aad2-a8feec644e96', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '128e870c-1644-40c4-985e-8b76cf8c3156', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '2d06bdcb-ce81-4dc7-ac30-3ccae43c0e06', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '74d078bf-2da2-4dd8-8eb5-ff9a763a3b88', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c5ee8b88-e4b1-4b0f-9d75-ca5243d3dd2f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9d80efa9-9e06-4cbc-98c1-6a8243250a6e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2900acea-86b6-472c-b969-f99bbcc9cfe4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '45583073-5c04-4992-96cc-cccbc39e5cf5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'ec5e4bdb-4aca-4230-92c9-5d0b2ab0dce5', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '27e07b25-4436-485f-b1cd-808105b6f90f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'f66f4527-db16-49e7-b751-af1ac65df6dd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '73a45c13-9b4a-4216-ad29-159b77d75bac', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6f0a394f-be6e-4d58-a423-7641197d4b45', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4f01a56b-55f5-4b87-a168-076bfb73408c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '47006b6c-45d2-4ab9-a580-bffd07fece81', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6c1bef42-5307-4733-a8a2-f3e3cf31190f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '46962b98-6b44-4544-8d57-722e7d9307b1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'materialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6b986684-7997-4091-9f86-82375bc8ce8f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '380e2b18-8c3f-40b2-8ef9-08067ed84a8f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': 'aa3e2bb1-d6aa-4c99-bcc8-671191799f91', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '693bb5f2-e63e-4cec-9cd5-4e28413ca6de', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '449aaa32-3f45-4a9c-8098-79e1edebe950', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '256f54c7-c589-4450-8dae-f20873de1c33', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '44a9ab96-78df-478c-8b7f-f9ff07d84ac6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '703f1887-ed82-4b4d-bc4b-8d21cfb2ca9e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': 'cf228396-cde5-412e-a858-8367d89cb9fd', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2986b956-1dfc-4153-a41f-afd3b7489c57', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': 'c1a537cc-0b8d-4b63-9c6e-0f5c743ef448', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '2af0aebb-f6f0-40e5-9871-701924f16deb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsicatti'}, 'indexTerms': {}}\n", + "{'uuid': '393bd1bb-7165-451d-8437-258497a86ba6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1e1ea9c5-a47e-4e95-828b-cf04579d624c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c9508dc1-3e40-4999-a9fd-d25e8229bf8e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9e40260b-d0d5-46c2-8d3e-cec74713b16d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '457a29d9-9dc8-4f43-ae7c-86e2dd8a2aeb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '116aff45-b173-433b-a796-1ef6ba421904', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a7d424d5-8b3c-4c1b-8f31-23998346409e', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd4a9d78c-9355-473b-959f-2ed78550c483', 'type': 'records', 'data': {'dwc:basisOfRecord': 'materialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4ccc142e-d901-40ce-8c9d-4e206cf88e2b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': 'c66ae2ef-8557-4327-95f3-dc6e13c37822', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '490a9ef7-10da-4ae7-bfd2-79c868ae8a9d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd07a0095-42db-4127-9bda-7b28d881e344', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '330c93ad-0c82-4818-83ec-6945bf6c2808', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '18e8059f-5708-42e2-a973-0acace7a1faa', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '0b304d47-598b-48b8-8f00-73237902be9d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '0ce9c6fd-515a-489a-8add-4670d83c6077', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '940e7a47-c9aa-44fb-82d1-68773c57ddc8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a76e81a7-f3da-4857-961d-19833fd6b48c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1b18692f-00eb-47b3-82bc-0c19e590bf25', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e5c8a60f-25a3-455a-af06-87ed4d5ed8e0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e74bfde6-dc5c-469f-bfe3-cd9e12fff0da', 'type': 'records', 'data': {'dwc:basisOfRecord': 'materialSample'}, 'indexTerms': {}}\n", + "{'uuid': '280bb1a6-b4e0-4fdd-9b2f-9aaae566668d', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '891ba660-dbb5-49ae-ae89-1581a6509ccc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '1f4f98e1-09e0-4a83-b2df-49408e9fadb2', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': 'edf82fd3-9329-4512-b2e5-3703c5b43bb1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c99a4170-7d65-4ff4-8f61-6d3adfb58d81', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '67533590-cc52-4cb8-ae53-37075d38e6b1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '604dcc9b-e059-44f0-a442-ef4337912fe4', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '9025700c-123f-4c4f-a5d2-44605aeaefa7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '43591725-d66b-43e7-af31-1950a4f26aee', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bb0095fa-ecd7-45de-8ada-46801d72b3fe', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'c77726a1-07ed-4db1-8853-9dcec992bbf7', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '8b440b1b-f6f9-4e00-8723-e33e50513d73', 'type': 'records', 'data': {'dwc:basisOfRecord': 'materialSample'}, 'indexTerms': {}}\n", + "{'uuid': '56a54ee4-fed6-4b8a-9684-8fd0fa8cafa8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '3a789441-d3fe-4e70-ba78-df78d6776b7f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': 'b5ba7412-86d7-4e1d-a310-7d9be5271062', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '68e5265e-e703-42e8-9af5-729134dda9c0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3804bd07-1367-468c-aa2c-2d1510437170', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1ba1daa9-a532-48a9-9694-1cab2f226f6f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7eae3d76-0680-45b9-8e84-05524568832a', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '53c54fcf-213d-432c-a9f7-dd00334d4851', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '3c1516ec-3f2b-42f6-852c-ed8b1667bd1b', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '409615ad-25a4-4b39-8d15-5fb01febdffe', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2d23dc05-d184-4765-838e-3bed14b0d097', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'e3a87800-49bc-4c45-b1d2-eb84ada73acb', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'a551f092-755b-4a83-acc4-fd41195956b0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'materialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'd567ca4f-9aaa-4fa4-a0fc-c368903f1838', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsicatti'}, 'indexTerms': {}}\n", + "{'uuid': '6af1f12a-504a-4c3f-9ab1-5cc1ba715eb6', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6b614526-0369-43a8-8aa1-c9e334d59c17', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'bf0daf10-9083-463f-82e3-5e1e954f8b66', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '7e67d56b-013c-4692-955c-698ea7a80227', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '46f6b6f4-a64a-4773-b590-bff388845aaf', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '1decaa59-9997-4d7b-93c4-064863c38449', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': 'd01d23e9-bc8a-4387-91f4-972a5d486f57', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '89a5c759-c4c5-437d-b777-12d119c3d99c', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '2729d053-85a3-4f44-b534-cd63283dfad0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': 'a76ff801-acbd-4f40-8c27-a3ab42725f45', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': 'ebaa6460-03a1-4b36-b616-2c83bd4ec44f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '814caa3f-737e-4223-aa87-f48fea3d45af', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '6b8f976e-8d9a-4047-91db-490a5f64f99f', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'adbb97fd-115d-46ca-bc8c-4f18cb919d81', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': 'a03919c9-8555-4aa1-aaa9-fb76f3177206', 'type': 'records', 'data': {'dwc:basisOfRecord': 'Exsiccati'}, 'indexTerms': {}}\n", + "{'uuid': '4ae1359e-98e7-48fe-96ac-cec126fc7bcc', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4263dc64-97a9-40b9-8db3-9bd2126141e0', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '4e61fa09-a515-4d82-b6dc-db51ba0d0d31', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': '1e934c7b-bfc0-41ee-9964-6e1490715df8', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n", + "{'uuid': 'aa6bf1b6-774d-4146-8ce4-65a23b71b4c1', 'type': 'records', 'data': {'dwc:basisOfRecord': 'MaterialSample'}, 'indexTerms': {}}\n" + ] + } + ], + "source": [ + "for each in record_list[\"items\"]:\n", + " if \"dwc:basisOfRecord\" in each[\"data\"]:\n", + " if each[\"data\"][\"dwc:basisOfRecord\"] not in dont_count_list:\n", + " print (each)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "---" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "# known bad values in basisOfRecord (we are looking for new good values)\n", + "dont_count_list=['Occurrence','MaterialSample']\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here are some of the other values providers have used." + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'Plants of the Mono Lake Basin', 'Exsiccati', 'Plants of Colorado', 'Exsicatti', 'Native and Naturalized Flora of', 'FieldBook', 'Plants of Jefferson County Open', 'Photograph', 'materialSample'}\n" + ] + } + ], + "source": [ + "found_basisOfRecord_values = set()\n", + "for each in record_list[\"items\"]:\n", + " if \"dwc:basisOfRecord\" in each[\"data\"]:\n", + " if each[\"data\"][\"dwc:basisOfRecord\"] not in dont_count_list:\n", + " found_basisOfRecord_values.add(each[\"data\"][\"dwc:basisOfRecord\"])\n", + "\n", + "print(found_basisOfRecord_values)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From eec97ca26667863e31aeba55bb5cb351504b74f4 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 9 Aug 2021 20:51:21 -0400 Subject: [PATCH 132/175] add more etags to blacklist (#186) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To avoid these errors over and over. ``` Aug 09 19:46:20 c18node4 idigbio-ingestion[21955]: ERROR idb.deriv჻ 96e58bad25add2281a3bd879d182d734 caused BadImageError: Error loading image IOError('image file is truncated (21 bytes not processed)',) Aug 09 19:46:22 c18node4 idigbio-ingestion[21955]: ERROR idb.deriv჻ 0c5d6417183ce9f809bc4999dd039c3f caused BadImageError: Error loading image IOError('image file is truncated (18 bytes not processed)',) Aug 09 19:46:23 c18node4 idigbio-ingestion[21955]: ERROR idb.deriv჻ 893bdc41e61daa299223786f5328ea74 caused BadImageError: Error loading image IOError('image file is truncated (65 bytes not processed)',) Aug 09 19:46:24 c18node4 idigbio-ingestion[21955]: ERROR idb.deriv჻ a1eef457813cba1225c2d616ae3ca13c caused BadImageError: Error loading image IOError('image file is truncated (70 bytes not processed)',) Aug 09 19:46:25 c18node4 idigbio-ingestion[21955]: ERROR idb.deriv჻ 98d084e4f69d54f56ba5c394c8fe2867 caused BadImageError: Error loading image IOError('image file is truncated (80 bytes not processed)',) Aug 09 19:46:26 c18node4 idigbio-ingestion[21955]: ERROR idb.deriv჻ d6bc3794fdb66d072a0e6e4b2f18fbaf caused BadImageError: Error loading image IOError('image file is truncated (33 bytes not processed)',) Aug 09 19:46:27 c18node4 idigbio-ingestion[21955]: ERROR idb.deriv჻ 127ebdcaedfa2fb31b39fcf6eddeeab0 caused BadImageError: Error loading image IOError('image file is truncated (74 bytes not processed)',) Aug 09 19:46:28 c18node4 idigbio-ingestion[21955]: ERROR idb.deriv჻ e8e6a817aa26915c5534bd56191af63d caused BadImageError: Error loading image IOError('image file is truncated (27 bytes not processed)',) Aug 09 19:46:29 c18node4 idigbio-ingestion[21955]: ERROR idb.deriv჻ 61642488d9934c7e1c1b7a632ca57b8b caused BadImageError: Error loading image IOError('image file is truncated (82 bytes not processed)',) Aug 09 19:46:31 c18node4 idigbio-ingestion[21955]: ERROR idb.deriv჻ acc6ff0c6ab19388cb8110c052a80085 caused BadImageError: Error loading image IOError('image file is truncated (23 bytes not processed)',) Aug 09 19:46:32 c18node4 idigbio-ingestion[21955]: ERROR idb.deriv჻ 05ee826c5a834a9352f06c259dfed9c2 caused BadImageError: Error loading image IOError('image file is truncated (13 bytes not processed)',) Aug 09 19:46:33 c18node4 idigbio-ingestion[21955]: ERROR idb.deriv჻ 7a97f486c692a3534a39ac716f4c866e caused BadImageError: Error loading image IOError('image file is truncated (20 bytes not processed)',) Aug 09 19:46:34 c18node4 idigbio-ingestion[21955]: ERROR idb.deriv჻ b217d792a18a2cd1cf4b6f60b4fe8d67 caused BadImageError: Error loading image IOError('image file is truncated (11 bytes not processed)',) Aug 09 19:46:34 c18node4 idigbio-ingestion[21955]: INFO idb.deriv჻ Checked: 13 Generated: 0 Existed: 0 Erred: 13 Rate: 0.9/s ``` --- idb/blacklists/derivatives.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/idb/blacklists/derivatives.py b/idb/blacklists/derivatives.py index fa61208d..98d2b653 100644 --- a/idb/blacklists/derivatives.py +++ b/idb/blacklists/derivatives.py @@ -33,6 +33,7 @@ '052f17fd6a1d758cb7a058aa3a629545', '05bc648bec7f4d0f90493cdecef2aa33', '05e3041c0ee55331c355117c995000e4', + '05ee826c5a834a9352f06c259dfed9c2', '060712adcfa6f7b040f701ca03a266a0', '0628326e3e4e0fdd7699dd48831d4d05', '069752bd3600e47e1f52ae8b3198d728', @@ -69,6 +70,7 @@ '0bdab840876ff65808e60cefb200420a', '0c234361660b8ecfcca6a8bd45d3c445', '0c2cb9e37ea01d0a2a62b213c4eb5899', + '0c5d6417183ce9f809bc4999dd039c3f', '0c6cf45b5b98b535b7232ecb0043774c', '0c8daa545af86a8d3694c51b45989981', '0cbea54533054458364b597c3b888f47', @@ -99,6 +101,7 @@ '11d49a25f1600139188bc0d19defa2b6', '11f059f1872aae766a71995194e6b5af', '12523482232547122f2f4a56953b3893', + '127ebdcaedfa2fb31b39fcf6eddeeab0', '12b665c75d3e04cfd4c9fa548e02cf49', '1328643a680a06d2047fb24fe07af6a7', '13549b1d02a8f714ca1ada2cca96b18a', @@ -455,6 +458,7 @@ '60a379d8fe9b2daf9b76d73edfcc54e9', '613270e0f1f630b0034939bfa1058000', '6155cfe3e56f5c5cbe39eefb2f6f4c4d', + '61642488d9934c7e1c1b7a632ca57b8b', '616d783adbcc28efd193d216ae266ead', '6178dd680e125f4e7b25eb12dde72f4b', '61804ccb95db616851ce33a026b8a2cb', @@ -568,6 +572,7 @@ '7997118bfc8baf77d9e145b331ef1296', '7a15d318053d498451a0bca306f3e118', '7a7ceaf45da6ec49d0a3da6d1c2a8bd6', + '7a97f486c692a3534a39ac716f4c866e', '7a995acf71149384c6e0e7b94efcc8ea', '7a9d262dffbd40ec0e3563cf1ecbdaea', '7aa009a271f3f8f12b85424edb75194d', @@ -647,6 +652,7 @@ '88c889b7c284807cad16b01bc70a3407', '88f3b2721bff952eb16f0ebdab8aff34', '89041e1d6532461ea5290ecf7e63c9e9', + '893bdc41e61daa299223786f5328ea74', '89488afcd36ac32c229dad04fbfc60f9', '89e6ab19dfbf51aef39a36878256d937', '8a1d2c6f04d33de4de0ff0808ef93287', @@ -709,6 +715,7 @@ '965f0f52e11622d497f25deb3d2c9ceb', '969310ebb630b2bd582e66a40778f31e', '96a79c335d0141fba7f36f2c4d40452e', + '96e58bad25add2281a3bd879d182d734', '972a908e51bc810920b6e0f0faa03f39', '97419977b29bcf7072fee7023a940834', '97e4dd20338e334e174052a5290953f5', @@ -716,6 +723,7 @@ '98541f1ab176eff543c32aee7e3521d7', '98a5765ef891428900492b189458f55a', '98b3f2bbc5bec9590b126b61a657585a', + '98d084e4f69d54f56ba5c394c8fe2867', '99025f3b4978e2d75c6fba6f0bd295b2', '990b13c991d336897b0a35ba96c11fdf', '99674e03dd1b404963437cc4db8200cd', @@ -769,6 +777,7 @@ 'a176053c9f21ca729de393429ed835dd', 'a19c57a191b182e614a1ab3002d46a00', 'a1aec2fd6ccb57ff4d264838bf2702e6', + 'a1eef457813cba1225c2d616ae3ca13c', 'a214aa6ee2e6c3ca82516f4115811a5e', 'a2bae7aa69b283340714a70ca5191249', 'a2bcb8b69acb92a3fdb7074c4db149ba', @@ -816,6 +825,7 @@ 'ac12ef0c9039ba99201430c56fca5760', 'ac2a8597bf26093cdff8a95d38ec1657', 'aca1a6e5ee4f52d7651d664b88dab486', + 'acc6ff0c6ab19388cb8110c052a80085', 'aced115f2620426cb05eeefd56fa48c4', 'ad368531805ff9dcb8afa357a52506f5', 'ada26aab0b55bc5325c398c3eb7c3658', @@ -835,6 +845,7 @@ 'b14f07fa654ecf4445bb47e76280cf5c', 'b19e199b63ec110471db997701f5f69f', 'b1ae7dd1af59b1a5a35ac610cfe38975', + 'b217d792a18a2cd1cf4b6f60b4fe8d67', 'b22bc3afac5b3e42ada8c2a107c3a7b3', 'b28f9d7236b2b86cb185e78cb11d75d5', 'b3317fff300745ebe79cce6ad4f9ffdd', @@ -992,6 +1003,7 @@ 'd628ffd8aee1d8dadaf1864d295f3445', 'd68eab15bce2dc4af3a92fa815f59336', 'd6ada30b4cb916b077d618a73a93cf04', + 'd6bc3794fdb66d072a0e6e4b2f18fbaf', 'd6cc7d0e0120a6a267adac3a05f6f287', 'd6d08f04d065382933283a788f52f080', 'd70da1d4fd5fe85e82e660c183da108e', @@ -1067,6 +1079,7 @@ 'e84b60c31ac8b8b6d7b645714628448d', 'e88e2925289ee0ff74e70ac17486eecb', 'e8d35f97433156519ecfab0f1843b335', + 'e8e6a817aa26915c5534bd56191af63d', 'e8f8adeaa0c6687eb5edd1c075ae71f6', 'e90346599499fd027888748e592178c9', 'e93380c7ef8dd1571638a3fe0b7f256e', From a7a79742e4f6634e16de9fad29453d613676baba Mon Sep 17 00:00:00 2001 From: Ronald Canepa Date: Tue, 22 Jun 2021 16:16:28 -0400 Subject: [PATCH 133/175] [skip travis-ci] re-enable the push of stats into elastic by default --- idb/stats/collect.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/idb/stats/collect.py b/idb/stats/collect.py index 17146221..eb2ac1ee 100644 --- a/idb/stats/collect.py +++ b/idb/stats/collect.py @@ -118,6 +118,18 @@ def collect_stats(collect_datetime, es=None): LEFT JOIN queries on stats.query_id=queries.id WHERE date > %s AND date < %s """ + logger.debug("min is %s, max is %s, query is: %s" % (date_min, date_max, sql)) + + filename = "telem-output-structures-%s.log" % (datetime.now().strftime("%Y-%m-%d-%H-%M-%S")) + tracefilename = "trace-log-%s.log" % (datetime.now().strftime("%Y-%m-%d-%H-%M-%S")) + + # collects the raw objects we're sending to ES + output_fh = open(filename, 'w') + + trace_fh = open(tracefilename, 'w') + + trace_fh.write("doingquery and looping through results...") + for r in statsdbpool.fetchiter(sql, (date_min, date_max), cursor_factory=DictCursor): record_type = r["record_type"] stats_type = r["type"] @@ -177,8 +189,18 @@ def collect_stats(collect_datetime, es=None): "geo": json.loads(k), "count": recordset_stats[recordset_key][record_type][stat_type]["geocodes"][k] }) +<<<<<<< HEAD es.index(index=indexName, doc_type=typeName, body=recordset_data) +======= + trace_fh.write("writing recordset_data to ES: \n") + output_fh.write(json.dumps(recordset_data)) + trace_fh.write(json.dumps(recordset_data)) + + es.index(index=indexName, doc_type=typeName, body=recordset_data) + + logger.info("end of script") +>>>>>>> d5e4775 ([skip travis-ci] re-enable the push of stats into elastic by default) def api_stats(es=None): From db959a541dcc25750af8cebc57a83af39eab6fdc Mon Sep 17 00:00:00 2001 From: Ronald Canepa Date: Tue, 22 Jun 2021 15:31:26 -0400 Subject: [PATCH 134/175] change recordset stats aggregation job to use custom config file and marked old systemd files as deprecated --- ...tats.service => idb-api-stats.service-deprecated} | 2 ++ ...pi-stats.timer => idb-api-stats.timer-deprecated} | 2 ++ ....service => idb-collect-stats.service-deprecated} | 2 ++ ...tats.timer => idb-collect-stats.timer-deprecated} | 2 ++ .../system/idb-recordset-stats-aggregator.service | 12 ++++++++++++ 5 files changed, 20 insertions(+) rename etc/systemd/system/{idb-api-stats.service => idb-api-stats.service-deprecated} (63%) rename etc/systemd/system/{idb-api-stats.timer => idb-api-stats.timer-deprecated} (59%) rename etc/systemd/system/{idb-collect-stats.service => idb-collect-stats.service-deprecated} (63%) rename etc/systemd/system/{idb-collect-stats.timer => idb-collect-stats.timer-deprecated} (63%) create mode 100644 etc/systemd/system/idb-recordset-stats-aggregator.service diff --git a/etc/systemd/system/idb-api-stats.service b/etc/systemd/system/idb-api-stats.service-deprecated similarity index 63% rename from etc/systemd/system/idb-api-stats.service rename to etc/systemd/system/idb-api-stats.service-deprecated index 04d96859..754f5143 100644 --- a/etc/systemd/system/idb-api-stats.service +++ b/etc/systemd/system/idb-api-stats.service-deprecated @@ -1,3 +1,5 @@ +# deprecated for both naming and the call. see: idb-recordset-stats--aggregator.* + [Unit] Description="Runs `idb api-stats`" diff --git a/etc/systemd/system/idb-api-stats.timer b/etc/systemd/system/idb-api-stats.timer-deprecated similarity index 59% rename from etc/systemd/system/idb-api-stats.timer rename to etc/systemd/system/idb-api-stats.timer-deprecated index 2d3fc81d..256189b3 100644 --- a/etc/systemd/system/idb-api-stats.timer +++ b/etc/systemd/system/idb-api-stats.timer-deprecated @@ -1,3 +1,5 @@ +# deprecated for both naming and the call. see: idb-recordset-stats--aggregator.* + [Unit] Description="Daily timer for idb api-stats" diff --git a/etc/systemd/system/idb-collect-stats.service b/etc/systemd/system/idb-collect-stats.service-deprecated similarity index 63% rename from etc/systemd/system/idb-collect-stats.service rename to etc/systemd/system/idb-collect-stats.service-deprecated index 88cc1e73..ded4bcec 100644 --- a/etc/systemd/system/idb-collect-stats.service +++ b/etc/systemd/system/idb-collect-stats.service-deprecated @@ -1,3 +1,5 @@ +# deprecated for both naming and the call. see: idb-interaction-telemetry-aggregator.* + [Unit] Description="Runs `idb collect-stats`" diff --git a/etc/systemd/system/idb-collect-stats.timer b/etc/systemd/system/idb-collect-stats.timer-deprecated similarity index 63% rename from etc/systemd/system/idb-collect-stats.timer rename to etc/systemd/system/idb-collect-stats.timer-deprecated index 74eceb39..19cd83de 100644 --- a/etc/systemd/system/idb-collect-stats.timer +++ b/etc/systemd/system/idb-collect-stats.timer-deprecated @@ -1,3 +1,5 @@ +# deprecated for both naming and the call. see: idb-interaction-telemetry-aggregator.* + [Unit] Description="daily timer for idb collect-stats" diff --git a/etc/systemd/system/idb-recordset-stats-aggregator.service b/etc/systemd/system/idb-recordset-stats-aggregator.service new file mode 100644 index 00000000..d0206bb1 --- /dev/null +++ b/etc/systemd/system/idb-recordset-stats-aggregator.service @@ -0,0 +1,12 @@ +# https://redmine.idigbio.org/projects/infrastructure/wiki/Portal_and_api_stats_overview + +[Unit] +Description="Runs `idb api-stats` (recordset stats aggregator) via non-root user and virtualenv" + +[Service] +Type=oneshot +User=stats +Environment=LANG=en_US.UTF-8 + +# the virtualenv is "baked in" and does not need to be activated first +ExecStart=/home/stats/.virtualenvs/idb-backend/bin/python /home/stats/.virtualenvs/idb-backend/bin/idb --journal -vv --config /etc/idigbio/idigbio-recordset-stats-aggregator-overrides.json api-stats From 8201d572277b0f6e73cdfbfddcdabe6493b55a2c Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 23 May 2021 13:50:47 -0400 Subject: [PATCH 135/175] add "chrono:ChronometricAge" to list of un-indexable objects (#175) https://redmine.idigbio.org/issues/4366 --- idb/indexing/index_helper.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/idb/indexing/index_helper.py b/idb/indexing/index_helper.py index a3ed88cf..b8ee7639 100644 --- a/idb/indexing/index_helper.py +++ b/idb/indexing/index_helper.py @@ -9,7 +9,13 @@ from idb.helpers.logging import idblogger logger = idblogger.getChild('index_helper') -from urllib.parse import urlparse +# PYTHON3_WARNING +from urlparse import urlparse + + +# A problematic field name is "http://rs.iobis.org/obis/terms/measurementTypeID" inside +# the "obis:ExtendedMeasurementOrFact". +UNINDEXABLE_OBJECTS = ["obis:ExtendedMeasurementOrFact", "chrono:ChronometricAge"] def index_record(ei, rc, typ, r, do_index=True): """ From e5963dee4f681bbcee6a363f7eb249f647201e77 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Fri, 7 May 2021 19:50:12 -0400 Subject: [PATCH 136/175] truncate problematic objects containing dots in fieldnames (#173) * truncate problematic objects containing dots in fieldnames --- idb/indexing/index_helper.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/idb/indexing/index_helper.py b/idb/indexing/index_helper.py index b8ee7639..2f1760ca 100644 --- a/idb/indexing/index_helper.py +++ b/idb/indexing/index_helper.py @@ -56,8 +56,8 @@ def index_record(ei, rc, typ, r, do_index=True): g = grabAll(typ, d) i = prepForEs(typ, g) - # Remove fieldnames with dots in them - # to fix an issue converting to ES 2.3+ + # Fixup problematic field names due to limitations of Elasticsearch. + # For example, dots in fieldnames. for k in r["data"]: if "." in k: if k in types: @@ -72,6 +72,13 @@ def index_record(ei, rc, typ, r, do_index=True): suffix = urldata.path.split("/")[-1] r["data"][prefix + ":" + suffix] = r["data"][k] d["flag_data_" + prefix + "_" + suffix + "_munge"] = True + if k in UNINDEXABLE_OBJECTS: + # inventing a new flag for each field we are truncating + new_flag = "_".join(["flag", "idigbio", k.replace(":","_").lower(), "truncated"]) + d[new_flag] = True + # truncate the troublesome object to prevent Elasticsearch mapper_parsing_exception + d[k] = {} + r["data"][k] = {} i["data"] = r["data"] i["indexData"] = d From 01e3e68dc3239bba07f4e0a7fec9781cb0eeaec7 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 19 Apr 2021 22:14:46 -0400 Subject: [PATCH 137/175] add new ExtendedMeasurementOrFact extension (#167) --- idb/helpers/fieldnames.py | 1 + 1 file changed, 1 insertion(+) diff --git a/idb/helpers/fieldnames.py b/idb/helpers/fieldnames.py index a93857de..31e57662 100644 --- a/idb/helpers/fieldnames.py +++ b/idb/helpers/fieldnames.py @@ -54,6 +54,7 @@ ''' types = { "http://purl.org/NET/aec/associatedTaxa": {"shortname": "aec:associatedTaxa"}, + "http://rs.iobis.org/obis/terms/ExtendedMeasurementOrFact": {"shortname": "obis:ExtendedMeasurementOrFact"}, "http://rs.gbif.org/terms/1.0/Identifier": {"shortname": "gbif:Identifier"}, "http://rs.gbif.org/terms/1.0/Image": { "shortname": "dwc:Multimedia"}, "http://rs.gbif.org/terms/1.0/Multimedia": { "shortname": "dwc:Multimedia"}, From 27961f6901a35b2ca772e36f5b72689272565114 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Thu, 8 Apr 2021 23:12:52 -0400 Subject: [PATCH 138/175] add help for "idb datatables" (#155) Add cli help text for idb datatables. The existing sub-commands were actually junk in terms of functionality, so added some output so they are at least marginally useful. Also a few linting cleanups. Fixes #154 --- idb/data_tables/rights_strings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/idb/data_tables/rights_strings.py b/idb/data_tables/rights_strings.py index 839600c6..3dfbd57e 100644 --- a/idb/data_tables/rights_strings.py +++ b/idb/data_tables/rights_strings.py @@ -272,3 +272,4 @@ def main(): print() pp.pprint("*acceptable_licenses_trans*") pp.pprint(acceptable_licenses_trans) + From aebcf24459b6c0dcc84882e8a679a2b674d201dc Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 27 Feb 2022 18:57:02 -0500 Subject: [PATCH 139/175] fixup doc sentence --- tests/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/README.md b/tests/README.md index 2dd111d9..65ff6aaa 100644 --- a/tests/README.md +++ b/tests/README.md @@ -139,6 +139,8 @@ $ docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e "xpack One challenge is the test suite expects mappings, etc. to be present. +The ES 5.x image requires HTTP basic auth credentials. For example, to use curl with basic auth supply them via `-u` with the default username and password: + To do: 1. Have tests connect to localhost instead of the ES cluster that exists in CONFIG. From e7591f94ccb40026bc01a133b1231e41d8d94c26 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Sun, 27 Feb 2022 18:55:35 -0500 Subject: [PATCH 140/175] cherry pickin from python2 main --- tests/README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/README.md b/tests/README.md index 65ff6aaa..7591c64a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -141,11 +141,16 @@ One challenge is the test suite expects mappings, etc. to be present. The ES 5.x image requires HTTP basic auth credentials. For example, to use curl with basic auth supply them via `-u` with the default username and password: -To do: +``` +$ curl -u elastic:changeme http://localhost:9200/_cat/indices +``` + +TODO: -1. Have tests connect to localhost instead of the ES cluster that exists in CONFIG. -2. possibly pre-load a bunch of data / index. -3. possibly have that pre-loaded docker image available in docker-library +1. Change default password when running container +2. Have tests connect to localhost instead of the ES cluster that exists in CONFIG. +3. possibly pre-load a bunch of data / index. +4. possibly have that pre-loaded docker image available in docker-library ## Running All Tests From f9f3c9ca5fbc9ab8e91ba3c7959790683d5f37cd Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Thu, 10 Mar 2022 08:53:16 -0500 Subject: [PATCH 141/175] add "mappings" command to idb index to only put mappings (#211) --- idb/indexing/__init__.py | 14 ++++++++++++++ idb/indexing/index_from_postgres.py | 3 +++ 2 files changed, 17 insertions(+) diff --git a/idb/indexing/__init__.py b/idb/indexing/__init__.py index 53ba9287..b128fea9 100644 --- a/idb/indexing/__init__.py +++ b/idb/indexing/__init__.py @@ -186,3 +186,17 @@ def check(params): for k in params: params[k] = params[k]() resume(also_delete=True, **params) + + +@cli.command(help="PUT mappings only and then exit") +@click.pass_obj +@fnlogged +def mappings(params): + """ + We currently put the ES mappings whenever we initialize the connection to Elasticsearch. + This option will not attempt any actual indexing but will put the mappings into the index. + """ + for k in params: + params[k] = params[k]() + from .index_from_postgres import mappings_only + mappings_only(**params) diff --git a/idb/indexing/index_from_postgres.py b/idb/indexing/index_from_postgres.py index 8998e43b..59012e7f 100644 --- a/idb/indexing/index_from_postgres.py +++ b/idb/indexing/index_from_postgres.py @@ -286,6 +286,9 @@ def uuids(ei, rc, uuid_l, no_index=False, children=False): consume(ei, rc, f, no_index=no_index) ei.optimize() +def mappings_only(ei): + ei.optimze() + def consume(ei, rc, iter_func, no_index=False): for typ in ei.types: # Construct a version of index record that can be just called with the From e98a3ced60ae6cb69477cc539fbe4be557ab0b62 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 23 Mar 2022 18:24:49 -0400 Subject: [PATCH 142/175] cleanup tab in setup.py --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 1a81683f..dfd41cfe 100644 --- a/setup.py +++ b/setup.py @@ -50,8 +50,7 @@ def read(*paths): 'pytz>=2016.10', 'requests==2.20.0', 'urllib3<1.25,>=1.21.1', - 'pycrypto', - #'flask>=0.11.0, <1.0.0', + 'pycrypto', 'flask==1.1.2', 'Flask-UUID', 'Flask-CORS', From 484f67d12c05b3fa9e903e48a90a4c56e2d271b9 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 23 Mar 2022 18:26:03 -0400 Subject: [PATCH 143/175] suggest INFO for celery logging level --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9dcf47bd..87c1e2f5 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ This package can also be run as a [celery worker]; this is used by the data api that launches some tasks (most notably download building) via celery to a background worker. - celery worker --app=idigbio_workers -l WARNING + celery worker --app=idigbio_workers -l INFO [celery worker]: http://docs.celeryproject.org/en/latest/userguide/workers.html From 58c8121585d952c519cde3c56257f20c6e6b2eb9 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 23 Mar 2022 20:08:22 -0400 Subject: [PATCH 144/175] add comment about IPT guids and regex warning --- idigbio_ingestion/update_publisher_recordset.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/idigbio_ingestion/update_publisher_recordset.py b/idigbio_ingestion/update_publisher_recordset.py index c055661b..39d41927 100644 --- a/idigbio_ingestion/update_publisher_recordset.py +++ b/idigbio_ingestion/update_publisher_recordset.py @@ -80,7 +80,17 @@ def id_func(portal_url, e): portal_url, e["collid"]) if id is not None: - # Strip trailing version info from ipt ids + # Strip trailing version info from IPT ids. For example, if the RSS feed + # contains: + # 88d5d94e-f762-11e1-a439-00145eb45e9a/v2.68 + # we want to keep only the guid portion which stays the same for this entry + # over time. + # + # TODO: + # The python invalid escape sequence warning triggered by the following regex + # due to the backslash _should_ be fixable by using a raw string instead. + # r'^(.*)...' + # Not changing yet because we have zero code coverage here. m = re.search('^(.*)/v[0-9]*(\.)?[0-9]*$', id) if m is not None: id = m.group(1) From bdb113d52a97ff73a283e7bb2620b2a12dcb51fa Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 23 Mar 2022 20:34:11 -0400 Subject: [PATCH 145/175] add linter warning --- idigbio_ingestion/verify_ceph_objects.py | 1 + 1 file changed, 1 insertion(+) diff --git a/idigbio_ingestion/verify_ceph_objects.py b/idigbio_ingestion/verify_ceph_objects.py index 9a54ce0c..472f2937 100644 --- a/idigbio_ingestion/verify_ceph_objects.py +++ b/idigbio_ingestion/verify_ceph_objects.py @@ -46,6 +46,7 @@ def check_object(mo, store=STORE): k = store.get_key(mo.keyname, mo.bucketname) return CheckResult(mo, k.exists()) except S3ResponseError as e: + # WARNING: Exception.message removed in Python 3 (exception-message-attribute) return CheckResult(mo, e.message) except: logger.exception("Failed on %s", mo.etag) From b5427402b8c81bd80fc5480841805e515dd149e2 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Wed, 23 Mar 2022 21:07:29 -0400 Subject: [PATCH 146/175] add pylint notes --- idigbio_ingestion/mediaing/fetcher.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/idigbio_ingestion/mediaing/fetcher.py b/idigbio_ingestion/mediaing/fetcher.py index 7a7ab75e..fbc69850 100644 --- a/idigbio_ingestion/mediaing/fetcher.py +++ b/idigbio_ingestion/mediaing/fetcher.py @@ -45,6 +45,7 @@ def once(prefix=None, ignores=IGNORE_PREFIXES): logger.info("mediaing - run once mode") fetchitems = get_items(prefix=prefix) groups = group_by_prefix(fetchitems) + # pylint: dict.values referenced when not iterating (dict-values-not-iterating) procs = start_all_procs(groups).values() fetchitems = None groups = None @@ -111,6 +112,7 @@ def process_list(fetchitems, forprefix=''): fetchrpool = gevent.pool.Pool(get_fetcher_count(forprefix)) uploadpool = gevent.pool.Pool(8) items = fetchrpool.imap_unordered(lambda fi: fi.get_media(), fetchitems, maxsize=10) + # pylint: map built-in referenced when not iterating (map-builtin-not-iterating) items = uploadpool.imap_unordered(lambda fi: fi.upload_to_storage(store), items, maxsize=10) items = map(FetchItem.cleanup, items) items = update_db_status(items) From 8b191d7ce73bd8060e0e9a63bf72871640451d0a Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 6 Feb 2023 20:34:25 -0500 Subject: [PATCH 147/175] fix pytest and pip package builds, pin versions --- README.md | 2 +- setup.py | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 87c1e2f5..761da99a 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ iDigBio server and backend code for data ingestion and data API. In Ubuntu 18.04: - apt install python3-dev libblas-dev liblapack-dev \ + apt install python3-dev python3-all-dev libblas-dev liblapack-dev \ libatlas-base-dev gfortran libgdal-dev libpq-dev libgeos-c1v5 \ libsystemd-dev diff --git a/setup.py b/setup.py index dfd41cfe..0c020ba6 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,10 @@ def read(*paths): else: pillow_package = "pillow-simd>=3.4,<=5.1.1" + # setup_requires=["cffi>=1.0.0"], + # cffi_modules=["package/foo_build.py:ffibuilder"], + # install_requires=["cffi>=1.0.0"], + setup( name='idb-backend', version=version, @@ -39,6 +43,7 @@ def read(*paths): author_email='idigbio@acis.ufl.edu', packages=find_packages(exclude=['tests*']), setup_requires=['pytest-runner'], + # setup_requires=["pytest-runner","cffi>=1.0.0"], install_requires=[ 'idigbio>=0.8.2', 'psycopg2-binary>=2.8.3', @@ -63,7 +68,7 @@ def read(*paths): 'shapely', 'celery[redis]>=4.0, <4.3', 'boto>=2.39.0, <3.0.0', - 'fiona', + 'fiona~=1.8', 'python-magic>=0.4.11, <=0.5.0', 'feedparser>=5.2.0', 'click>=6.3, <7.0', @@ -72,19 +77,23 @@ def read(*paths): 'path.py>=10.0.0, <11', 'wsgi-request-logger>=0.4.6', 'jsonlines>=1.1.3', + 'wheel', + 'itsdangerous==2.0.1', + 'jinja2<3.1.0', + 'werkzeug==2.0.3' ], extras_require={ 'ingestion': [ 'pydub==0.16.5', pillow_package, 'lxml', - 'chardet', + 'chardet~=3.0', 'pyquery>=1.2', ], 'test': [ - 'pytest>=3.0', + 'pytest~=4.0', 'pytest-cov', - 'pytest-flask', + 'pytest-flask~=0.15', 'pytest-mock==1.13.0', 'fakeredis', ] From 19b9cc9f4875b335576557ed114833f718c4b8c1 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 7 Feb 2023 20:51:37 -0500 Subject: [PATCH 148/175] remove commented out section --- idb/helpers/conversions.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/idb/helpers/conversions.py b/idb/helpers/conversions.py index dbd5389f..fca62c92 100644 --- a/idb/helpers/conversions.py +++ b/idb/helpers/conversions.py @@ -879,34 +879,6 @@ def fixBOR(t, r): r["flag_dwc_basisofrecord_removed"] = True r["flag_dwc_basisofrecord_invalid"] = True - # Disable based on feedback from John W. and Joanna - # if r["basisofrecord"] == "preservedspecimen": - # paleo_terms = [ - # "bed", - # "group", - # "member", - # "formation", - # "lowestbiostratigraphiczone", - # "lithostratigraphicterms", - # "earliestperiodorlowestsystem", - # "earliesteraorlowesterathem", - # "earliestepochorlowestseries", - # "earliestageorloweststage", - # "latesteraorhighesterathem", - # "latestepochorhighestseries", - # "latestageorhigheststage", - # "latestperiodorhighestsystem", - # ] - - # for f in paleo_terms: - # if filled(f,r): - # r["flag_dwc_basisofrecord_paleo_conflict"] = True - # r["flag_dwc_basisofrecord_replaced"] = True - # r["basisofrecord"] = "fossilspecimen" - # break - else: - r["flag_dwc_basisofrecord_invalid"] = True - def fix_taxon_rank(t, r): if filled("taxonrank", r): if r["taxonrank"] in taxon_rank.mapping: From 3aea9feecbd4ca71ee3f8803141f1384505e7b5d Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 7 Feb 2023 20:54:32 -0500 Subject: [PATCH 149/175] dq flags now only get assigned to `records` --- idb/helpers/conversions.py | 70 +++++++++++++++------------ tests/idb/test_helpers_conversions.py | 42 +++++++++++++++- 2 files changed, 79 insertions(+), 33 deletions(-) diff --git a/idb/helpers/conversions.py b/idb/helpers/conversions.py index fca62c92..59f97765 100644 --- a/idb/helpers/conversions.py +++ b/idb/helpers/conversions.py @@ -857,46 +857,50 @@ def fixBOR(t, r): These comparisons using lowercased version seem to work, even though supplied values generally match the class name in the standard. e.g. To find PreservedSpecimen we look for "preserved". """ - if filled("basisofrecord", r): - if "preserved" in r["basisofrecord"]: - r["basisofrecord"] = "preservedspecimen" - elif "fossil" in r["basisofrecord"]: - r["basisofrecord"] = "fossilspecimen" - elif "living" in r["basisofrecord"]: - r["basisofrecord"] = "livingspecimen" - elif "material" in r["basisofrecord"]: - r["basisofrecord"] = "materialsample" - elif "specimen" in r["basisofrecord"]: - r["basisofrecord"] = "preservedspecimen" - elif "machine" in r["basisofrecord"] and "observation" in r["basisofrecord"]: - r["basisofrecord"] = "machineobservation" - elif "observation" in r["basisofrecord"]: - r["basisofrecord"] = "humanobservation" - elif "occurrence" in r["basisofrecord"]: - r["basisofrecord"] = "occurrence" + if t == "records": + if filled("basisofrecord", r): + if "preserved" in r["basisofrecord"]: + r["basisofrecord"] = "preservedspecimen" + elif "fossil" in r["basisofrecord"]: + r["basisofrecord"] = "fossilspecimen" + elif "living" in r["basisofrecord"]: + r["basisofrecord"] = "livingspecimen" + elif "material" in r["basisofrecord"]: + r["basisofrecord"] = "materialsample" + elif "specimen" in r["basisofrecord"]: + r["basisofrecord"] = "preservedspecimen" + elif "machine" in r["basisofrecord"] and "observation" in r["basisofrecord"]: + r["basisofrecord"] = "machineobservation" + elif "observation" in r["basisofrecord"]: + r["basisofrecord"] = "humanobservation" + elif "occurrence" in r["basisofrecord"]: + r["basisofrecord"] = "occurrence" + else: + r["basisofrecord"] = None + r["flag_dwc_basisofrecord_removed"] = True + r["flag_dwc_basisofrecord_invalid"] = True else: - r["basisofrecord"] = None - r["flag_dwc_basisofrecord_removed"] = True r["flag_dwc_basisofrecord_invalid"] = True def fix_taxon_rank(t, r): - if filled("taxonrank", r): - if r["taxonrank"] in taxon_rank.mapping: - if taxon_rank.mapping[r["taxonrank"]] is None: + if t == "records": + if filled("taxonrank", r): + if r["taxonrank"] in taxon_rank.mapping: + if taxon_rank.mapping[r["taxonrank"]] is None: + r["taxonrank"] = None + r["flag_dwc_taxonrank_removed"] = True + r["flag_dwc_taxonrank_invalid"] = True + elif r["taxonrank"] != taxon_rank.mapping[r["taxonrank"]]: + r["taxonrank"] = taxon_rank.mapping[r["taxonrank"]] + r["flag_dwc_taxonrank_replaced"] = True + else: + pass # Taxon Rank is in the mapping as an identity. + elif r["taxonrank"] not in taxon_rank.acceptable: r["taxonrank"] = None r["flag_dwc_taxonrank_removed"] = True r["flag_dwc_taxonrank_invalid"] = True - elif r["taxonrank"] != taxon_rank.mapping[r["taxonrank"]]: - r["taxonrank"] = taxon_rank.mapping[r["taxonrank"]] - r["flag_dwc_taxonrank_replaced"] = True else: - pass # Taxon Rank is in the mapping as an identity. - elif r["taxonrank"] not in taxon_rank.acceptable: - r["taxonrank"] = None - r["flag_dwc_taxonrank_removed"] = True - r["flag_dwc_taxonrank_invalid"] = True - else: - pass # Taxon Rank is Acceptable, but not mapped + pass # Taxon Rank is Acceptable, but not mapped # Step, count, ms, ms/count action # rc 1000 354.179 0.354179 record corrector @@ -930,6 +934,8 @@ def grabAll(t, d): r.update(collect_genbank_sequences(t,d)) # Done with non-dependant fields. + # Only do corrections and set flags on specimen records! + #if t == "records": gs_sn_crossfill(t, r) fixBOR(t, r) fix_taxon_rank(t, r) diff --git a/tests/idb/test_helpers_conversions.py b/tests/idb/test_helpers_conversions.py index 73024e43..8f1d9fbb 100644 --- a/tests/idb/test_helpers_conversions.py +++ b/tests/idb/test_helpers_conversions.py @@ -273,7 +273,7 @@ def test_scientific_name_filler(self): class TestGrabAll(unittest.TestCase): - def test_grab_all(self): + def test_grab_all_on_record(self): r = { "idigbio:uuid": "0000012b-9bb8-42f4-ad3b-c958cb22ae45", "idigbio:etag": "cb7d64ec3aef36fa4dec6a028b818e331a67aacc", @@ -396,6 +396,46 @@ def test_grab_all(self): self.assertEqual(output['hasImage'], True) self.assertEqual(output['hasMedia'], True) + def test_grab_all_on_mediarecord(self): + r = { + "uuid": "256098cf-723d-4633-a827-7fbf60a103ec", + "type": "mediarecords", + "etag": "5da26d3ee501516557689e55e1b1dd6c195414a2", + "data": { + "dcterms:type": "StillImage", + "ac:providerManagedID": "urn:uuid:21943c26-e2d2-49d8-8626-18f1d5eeb56d", + "ac:subtype": "Photograph", + "ac:metadataLanguage": "en", + "xmpRights:UsageTerms": "CC BY-NC-SA (Attribution-NonCommercial-ShareAlike)", + "ac:thumbnailAccessURI": "https://bryophyteportal.org/imglib/storage/srp/bryophytes/SRP-B-0000/SRP-B-0000026_tn.jpg", + "dcterms:format": "image/jpeg", + "ac:goodQualityAccessURI": "https://bryophyteportal.org/imglib/storage/srp/bryophytes/SRP-B-0000/SRP-B-0000026.JPG", + "coreid": "2212837", + "dcterms:identifier": "https://bryophyteportal.org/imglib/storage/srp/bryophytes/SRP-B-0000/SRP-B-0000026_lg.jpg", + "xmpRights:Owner": "Boise State University Lichen Herbarium (SRP)", + "dcterms:rights": "http://creativecommons.org/licenses/by-nc/3.0/", + "ac:accessURI": "https://bryophyteportal.org/imglib/storage/srp/bryophytes/SRP-B-0000/SRP-B-0000026_lg.jpg", + "xmp:MetadataDate": "2013-06-05 23:10:50", + "ac:associatedSpecimenReference": "https://bryophyteportal.org/portal/collections/individual/index.php?occid=2212837" + } + } + d = copy.deepcopy(r["data"]) + output = conversions.grabAll("mediarecords", d) + self.assertListEqual([],output['flags']) + + def test_grab_all_on_publisher(self): + r = { + "auto_publish": "false", + "base_url": "null", + "name": "FCC Tardigrades Darwin Core Archive rss feed", + "publisher_type": "rss", + "recordsets": {}, + "rss_url": "https://mywaterbears.org/portal/content/dwca/rss.xml" + } + + output = conversions.grabAll("publishers", r) + self.assertListEqual([],output['flags']) + class TestGetfield(unittest.TestCase): def test_getfield(self): From 18bcd0209db32f8e2a8ac7e9511b02c9ac9fede3 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Thu, 9 Feb 2023 19:24:16 -0500 Subject: [PATCH 150/175] non-records types *do* get dqs and flags fields --- idb/helpers/conversions.py | 4 +--- tests/idb/test_helpers_conversions.py | 5 ++++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/idb/helpers/conversions.py b/idb/helpers/conversions.py index 59f97765..843d8acb 100644 --- a/idb/helpers/conversions.py +++ b/idb/helpers/conversions.py @@ -934,8 +934,7 @@ def grabAll(t, d): r.update(collect_genbank_sequences(t,d)) # Done with non-dependant fields. - # Only do corrections and set flags on specimen records! - #if t == "records": + gs_sn_crossfill(t, r) fixBOR(t, r) fix_taxon_rank(t, r) @@ -952,7 +951,6 @@ def grabAll(t, d): if k.startswith("flag_"): r["flags"].append("_".join(k.split("_")[1:])) r["dqs"] = score(t, r) - return r diff --git a/tests/idb/test_helpers_conversions.py b/tests/idb/test_helpers_conversions.py index 8f1d9fbb..6fa6b198 100644 --- a/tests/idb/test_helpers_conversions.py +++ b/tests/idb/test_helpers_conversions.py @@ -422,6 +422,8 @@ def test_grab_all_on_mediarecord(self): d = copy.deepcopy(r["data"]) output = conversions.grabAll("mediarecords", d) self.assertListEqual([],output['flags']) + self.assertGreaterEqual(output['dqs'], 0.0) + self.assertLessEqual(output['dqs'], 1.0) def test_grab_all_on_publisher(self): r = { @@ -435,7 +437,8 @@ def test_grab_all_on_publisher(self): output = conversions.grabAll("publishers", r) self.assertListEqual([],output['flags']) - + self.assertGreaterEqual(output['dqs'], 0.0) + self.assertLessEqual(output['dqs'], 1.0) class TestGetfield(unittest.TestCase): def test_getfield(self): From 57977b6797ef2a55c0071ecd6232ec0b9dde2abe Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Thu, 9 Feb 2023 19:40:03 -0500 Subject: [PATCH 151/175] use postgres version 11 when testing with docker --- tests/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/README.md b/tests/README.md index 7591c64a..41da93d2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -116,14 +116,14 @@ $ ssh -nNT -L 9200:ELASTICSEARCH_CLUSTER_NODE_IP:9200 USER@SSH_HOST ``` -The local postgresql 9.5 DB is named `test_idigbio` with user/pass `test` / `test`. +The local postgresql DB is named `test_idigbio` with user/pass `test` / `test`. Note: The data in the db with that name will be destroyed during testing. A temporary instance of postgres running in docker will suffice: ``` -$ docker run --rm --name postgres_test_idigbio --network host -e POSTGRES_PASSWORD=test -e POSTGRES_USER=test -e POSTGRES_DB=test_idigbio -d postgres:9.5 +$ docker run --rm --name postgres_test_idigbio --network host -e POSTGRES_PASSWORD=test -e POSTGRES_USER=test -e POSTGRES_DB=test_idigbio -d postgres:11 ``` ### WIP: run Elasticsearch in local docker the same way we run postgres From fac0b6bbe5eae24216a116663a32dc109635c3f9 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 28 Feb 2023 19:48:15 -0500 Subject: [PATCH 152/175] pin very specific version of fiona for compatibility with older click --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0c020ba6..4a31b992 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ def read(*paths): 'shapely', 'celery[redis]>=4.0, <4.3', 'boto>=2.39.0, <3.0.0', - 'fiona~=1.8', + 'fiona==1.8.20', 'python-magic>=0.4.11, <=0.5.0', 'feedparser>=5.2.0', 'click>=6.3, <7.0', From 43c5ab02c16a05c52bfbf400f57d3586e3f0c2ec Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 28 Feb 2023 20:00:39 -0500 Subject: [PATCH 153/175] bump python3 version --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 03805e28..a3c03e39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ sudo: required dist: xenial language: python -python: 3.6.9 +python: 3.8.10 services: - docker - postgresql From d3994af2be1dd22c34d722d962b9443a1396fafd Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 28 Feb 2023 20:25:03 -0500 Subject: [PATCH 154/175] bump python3 to 3.8.10 in Dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e0ae7da6..14b19983 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -FROM python:3.6.9 -# python:3.6.9 is based on "Debian GNU/Linux 10 (buster)" +FROM python:3.8.10 +# since python:3.6 is based on "Debian GNU/Linux 10 (buster)" RUN apt-get update && apt-get install -y --no-install-recommends \ gfortran \ libatlas-base-dev \ From 7ea81bdcd901e27131dcd5c77f1480674e2a6d7c Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 27 Mar 2023 20:22:18 -0400 Subject: [PATCH 155/175] clarify comment --- Dockerfile | 2 +- idigbio_ingestion/cli.py | 8 ++++---- idigbio_ingestion/db_check.py | 12 ++++++------ idigbio_ingestion/db_rsids.py | 4 ++-- idigbio_ingestion/ds_sum_counts.py | 12 ++++++------ idigbio_ingestion/lib/delimited.py | 8 ++++---- idigbio_ingestion/lib/dwca.py | 2 +- idigbio_ingestion/lib/eml.py | 4 ++-- idigbio_ingestion/lib/fileproxy.py | 6 +++--- idigbio_ingestion/lib/waveform.py | 6 +++--- idigbio_ingestion/lib/xmlDictTools.py | 10 +++++----- idigbio_ingestion/mediaing/__init__.py | 2 +- idigbio_ingestion/mediaing/cli.py | 2 +- idigbio_ingestion/mediaing/derivatives.py | 10 +++++----- idigbio_ingestion/mediaing/fetcher.py | 8 ++++---- idigbio_ingestion/mediaing/migrate.py | 2 +- idigbio_ingestion/mediaing/updatedb.py | 4 ++-- idigbio_ingestion/recordset_cleanup.py | 2 +- idigbio_ingestion/verify_ceph_objects.py | 4 ++-- 19 files changed, 54 insertions(+), 54 deletions(-) diff --git a/Dockerfile b/Dockerfile index 14b19983..30475820 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM python:3.8.10 -# since python:3.6 is based on "Debian GNU/Linux 10 (buster)" +# python:3.6 and later are based on "Debian GNU/Linux" RUN apt-get update && apt-get install -y --no-install-recommends \ gfortran \ libatlas-base-dev \ diff --git a/idigbio_ingestion/cli.py b/idigbio_ingestion/cli.py index 8094ec4c..c5d056a0 100644 --- a/idigbio_ingestion/cli.py +++ b/idigbio_ingestion/cli.py @@ -1,5 +1,5 @@ -from __future__ import division, absolute_import -from __future__ import print_function + + import click import json @@ -33,7 +33,7 @@ def manual_update_recordset_from_file(rsid, file): @click.argument("rsid", type=click.UUID) @fnlogged def db_check(rsid): - rsid = u'{0}'.format(rsid) + rsid = '{0}'.format(rsid) from idigbio_ingestion.db_check import main main(rsid, ) @@ -75,7 +75,7 @@ def db_check_all(since): @click.argument("rsid", type=click.UUID) @fnlogged def ingest(rsid): - rsid = u'{0}'.format(rsid) + rsid = '{0}'.format(rsid) from idigbio_ingestion.db_check import main main(rsid, ingest=True) diff --git a/idigbio_ingestion/db_check.py b/idigbio_ingestion/db_check.py index 7e2707a9..d83fd6f6 100644 --- a/idigbio_ingestion/db_check.py +++ b/idigbio_ingestion/db_check.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import, print_function + import datetime import functools import json @@ -30,7 +30,7 @@ from idigbio_ingestion.lib.delimited import DelimitedFile -bad_chars = u"\ufeff" +bad_chars = "\ufeff" bad_char_re = re.compile("[%s]" % re.escape(bad_chars)) logger = idblogger.getChild("db-check") @@ -301,11 +301,11 @@ def process_subfile(rf, rsid, rs_uuid_etag, rs_id_uuid, ingest=False, db=None): else: if config.IDB_EXTRA_SERIOUS_DEBUG == 'yes': rlogger.debug("Setting sibling for '{0}'".format(u)) - db.set_record(u, typ[:-1], rsid, r, [*ids_to_add.keys()], siblings) + db.set_record(u, typ[:-1], rsid, r, [*list(ids_to_add.keys())], siblings) ingestions += 1 elif ingest and deleted: db.undelete_item(u) - db.set_record(u, typ[:-1], rsid, r, ids_to_add.keys(), siblings) + db.set_record(u, typ[:-1], rsid, r, list(ids_to_add.keys()), siblings) resurrections += 1 @@ -551,7 +551,7 @@ def metadataToSummaryJSON(rsid, metadata, writeFile=True, doStats=True): no_recordid_count = 0 duplicate_record_count = 0 duplicate_id_count = 0 - for t in metadata["counts"].values(): + for t in list(metadata["counts"].values()): csv_line_count += t["total_line_count"] no_recordid_count += t["no_recordid_count"] duplicate_record_count += t["duplicate_record_count"] @@ -668,7 +668,7 @@ def launch_child(rsid, ingest): # Close any logging filehandlers on root, leave alone any # other stream handlers (e.g. stderr) this way main can set up # its own filehandler to `$RSID.db_check.log` - for fh in filter(lambda h: isinstance(h, logging.FileHandler), logging.root.handlers): + for fh in [h for h in logging.root.handlers if isinstance(h, logging.FileHandler)]: logging.root.removeHandler(fh) fh.close() diff --git a/idigbio_ingestion/db_rsids.py b/idigbio_ingestion/db_rsids.py index 93c4c30c..b70639c5 100644 --- a/idigbio_ingestion/db_rsids.py +++ b/idigbio_ingestion/db_rsids.py @@ -38,7 +38,7 @@ def main3(): def main2(): for r in PostgresDB().get_type_list("recordset", limit=None): try: - print (r["uuid"]) + print((r["uuid"])) except: traceback.print_exc() @@ -48,7 +48,7 @@ def main1(): o = r.json() for rs in o["items"]: - print (rs["uuid"]) + print((rs["uuid"])) if __name__ == '__main__': main3() diff --git a/idigbio_ingestion/ds_sum_counts.py b/idigbio_ingestion/ds_sum_counts.py index 7c75f1ab..8e563de7 100644 --- a/idigbio_ingestion/ds_sum_counts.py +++ b/idigbio_ingestion/ds_sum_counts.py @@ -1,8 +1,8 @@ """New version of ds_sum_counts that generates both summary and suspects """ -from __future__ import division, absolute_import -from __future__ import print_function + + import json import os @@ -38,16 +38,16 @@ def read_all_files(base): def write_header(fp): - fp.write(u",".join(header)) - fp.write(u"\n") + fp.write(",".join(header)) + fp.write("\n") def write_row(fp, row, k=None): if k is None: k = row['filename'] values = [str(row.get(fld, 0)) for fld in fields] - fp.write(k + u"," + u",".join(values)) - fp.write(u"\n") + fp.write(k + "," + ",".join(values)) + fp.write("\n") def is_row_suspect(row): diff --git a/idigbio_ingestion/lib/delimited.py b/idigbio_ingestion/lib/delimited.py index 81365232..baca7efa 100644 --- a/idigbio_ingestion/lib/delimited.py +++ b/idigbio_ingestion/lib/delimited.py @@ -32,7 +32,7 @@ def __init__(self, name, lineNumber, lineLength, lineArr): def flag_unicode_error(e): bad_chars = "".join([hex(ord(c)) for c in e.object[e.start:e.end]]) - return (u"DECODEERROR:" + bad_chars, e.end) + return ("DECODEERROR:" + bad_chars, e.end) codecs.register_error("flag_error", flag_unicode_error) @@ -80,7 +80,7 @@ def __init__(self, fh, encoding="utf8", delimiter=",", fieldenc="\"", header=Non t = defaultdict(int) if header is not None: self.fields = header - for k, v in header.items(): + for k, v in list(header.items()): cn = get_canonical_name(v) t[cn[1]] += 1 else: @@ -94,7 +94,7 @@ def __init__(self, fh, encoding="utf8", delimiter=",", fieldenc="\"", header=Non self.fields[k] = cn[0] if self.rowtype is None: - items = t.items() + items = list(t.items()) items.sort(key=lambda item: (item[1], item[0]), reverse=True) self.rowtype = items[0][0] self.logger.info("Setting row type to %s", self.rowtype) @@ -119,7 +119,7 @@ def close(self): """ self.filehandle.close() - def next(self): + def __next__(self): """ Returns the next line in the record file, used for iteration """ diff --git a/idigbio_ingestion/lib/dwca.py b/idigbio_ingestion/lib/dwca.py index 7c0e349b..4291e75f 100644 --- a/idigbio_ingestion/lib/dwca.py +++ b/idigbio_ingestion/lib/dwca.py @@ -192,7 +192,7 @@ def __init__(self,filedict,fh,logname=None): # e.g. dwc:basisOfRecord # Sort by longest namespaces first ns_found = False - for ns in sorted(namespaces.keys(),key=lambda x: len(x), reverse=True): + for ns in sorted(list(namespaces.keys()),key=lambda x: len(x), reverse=True): if term.startswith(ns): ns_found = True term = term.replace(ns,namespaces[ns]+":") diff --git a/idigbio_ingestion/lib/eml.py b/idigbio_ingestion/lib/eml.py index 2cadc64d..5a80356a 100644 --- a/idigbio_ingestion/lib/eml.py +++ b/idigbio_ingestion/lib/eml.py @@ -1,4 +1,4 @@ -from __future__ import print_function + from pyquery import PyQuery as pq from idb.data_tables.rights_strings import acceptable_licenses_trans @@ -138,7 +138,7 @@ def parseEml(id, emlFilename): seen_emails.append(contact["email"]) elif cc.tag == "positionName": contact["role"] = cc.text - if len(contact.keys()) > 0: + if len(list(contact.keys())) > 0: collection["contacts"].append(contact) collection["other_guids"] = [] diff --git a/idigbio_ingestion/lib/fileproxy.py b/idigbio_ingestion/lib/fileproxy.py index e039129d..c5457ae3 100644 --- a/idigbio_ingestion/lib/fileproxy.py +++ b/idigbio_ingestion/lib/fileproxy.py @@ -57,8 +57,8 @@ def fileno(self): def isatty(self): return self.__file.isatty() - def next(self): - x = self.__file.next() + def __next__(self): + x = next(self.__file) self.__read_chars += len(x) return x @@ -117,7 +117,7 @@ def main(): except: traceback.print_exc() x = inf.dump() - print (len(x), x) + print((len(x), x)) if __name__ == "__main__": main() diff --git a/idigbio_ingestion/lib/waveform.py b/idigbio_ingestion/lib/waveform.py index dfadc3d8..89e35d91 100644 --- a/idigbio_ingestion/lib/waveform.py +++ b/idigbio_ingestion/lib/waveform.py @@ -1,6 +1,6 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import division + + + import datetime import os.path diff --git a/idigbio_ingestion/lib/xmlDictTools.py b/idigbio_ingestion/lib/xmlDictTools.py index 405ab6a2..eddf6a01 100755 --- a/idigbio_ingestion/lib/xmlDictTools.py +++ b/idigbio_ingestion/lib/xmlDictTools.py @@ -90,7 +90,7 @@ def d2xml(d): @return: A etree Root element """ def _d2xml(d, p): - for k,v in d.items(): + for k,v in list(d.items()): if k.startswith("#"): p.set(k[1:],v) elif isinstance(v,dict): @@ -109,7 +109,7 @@ def _d2xml(d, p): if '!namespaces' in d: nsmap = d['!namespaces'] del d['!namespaces'] - k,v = d.items()[0] + k,v = list(d.items())[0] if nsmap: d['!namespaces'] = nsmap node = etree.Element(k, nsmap=nsmap) @@ -129,7 +129,7 @@ def _d2xml(d, p): # print Z # assert X == Z - from StringIO import StringIO + from io import StringIO x = etree.parse(StringIO(""" A @@ -144,7 +144,7 @@ def _d2xml(d, p): C """)) - print(xml2d(x.getroot())) + print((xml2d(x.getroot()))) test2 = """ @@ -267,4 +267,4 @@ def _d2xml(d, p): """ x = etree.parse(StringIO(test2)) - print(xml2d(x.getroot())) + print((xml2d(x.getroot()))) diff --git a/idigbio_ingestion/mediaing/__init__.py b/idigbio_ingestion/mediaing/__init__.py index 65ca5335..8fe24eea 100644 --- a/idigbio_ingestion/mediaing/__init__.py +++ b/idigbio_ingestion/mediaing/__init__.py @@ -1,4 +1,4 @@ -from __future__ import division, absolute_import, print_function + from enum import Enum diff --git a/idigbio_ingestion/mediaing/cli.py b/idigbio_ingestion/mediaing/cli.py index e520708d..6fd4ee0c 100644 --- a/idigbio_ingestion/mediaing/cli.py +++ b/idigbio_ingestion/mediaing/cli.py @@ -1,4 +1,4 @@ -from __future__ import division, absolute_import, print_function + import click from gevent import monkey diff --git a/idigbio_ingestion/mediaing/derivatives.py b/idigbio_ingestion/mediaing/derivatives.py index f99128e0..725e642f 100644 --- a/idigbio_ingestion/mediaing/derivatives.py +++ b/idigbio_ingestion/mediaing/derivatives.py @@ -1,5 +1,5 @@ -from __future__ import division, absolute_import -from __future__ import print_function + + from io import BytesIO @@ -84,7 +84,7 @@ def one(o): ci = get_keys(o) gr = generate_all(ci) return upload_all(gr) - results = pool.imap_unordered(one, filter(None, objects)) + results = pool.imap_unordered(one, [_f for _f in objects if _f]) results = count_results(results, update_freq=100) etags = ((gr.etag,) for gr in results if gr) count = apidbpool.executemany( @@ -150,7 +150,7 @@ def get_keys(obj): etag, bucket = obj.etag, obj.bucket etag = str(etag) s = get_store() - bucketbase = u"idigbio-{0}-{1}".format(bucket, config.ENV) + bucketbase = "idigbio-{0}-{1}".format(bucket, config.ENV) mediakey = s.get_key(etag, bucketbase) keys = [s.get_key(etag + ".jpg", bucketbase + '-' + dtype) for dtype in DTYPES] return CheckItem(etag, bucket, mediakey, keys) @@ -177,7 +177,7 @@ def generate_all(item): try: - items = map(lambda k: build_deriv(item, img, k), item.keys) + items = [build_deriv(item, img, k) for k in item.keys] return GenerateResult(item.etag, list(items)) except BadImageError as bie: logger.error("%s: %s", item.etag, bie.message) diff --git a/idigbio_ingestion/mediaing/fetcher.py b/idigbio_ingestion/mediaing/fetcher.py index fbc69850..75d6f1c9 100644 --- a/idigbio_ingestion/mediaing/fetcher.py +++ b/idigbio_ingestion/mediaing/fetcher.py @@ -1,4 +1,4 @@ -from __future__ import division, absolute_import, print_function + import logging import itertools @@ -46,7 +46,7 @@ def once(prefix=None, ignores=IGNORE_PREFIXES): fetchitems = get_items(prefix=prefix) groups = group_by_prefix(fetchitems) # pylint: dict.values referenced when not iterating (dict-values-not-iterating) - procs = start_all_procs(groups).values() + procs = list(start_all_procs(groups).values()) fetchitems = None groups = None logger.debug("%d procs started, waiting...", len(procs)) @@ -82,7 +82,7 @@ def continuous(prefix=None, looptime=3600): logger.debug("Loop top") t1 = datetime.now() ignores = set(IGNORE_PREFIXES) - for pf, proc in running.items(): + for pf, proc in list(running.items()): if proc.exitcode is not None: del running[pf] lvl = logging.CRITICAL if proc.exitcode != 0 else logging.DEBUG @@ -114,7 +114,7 @@ def process_list(fetchitems, forprefix=''): items = fetchrpool.imap_unordered(lambda fi: fi.get_media(), fetchitems, maxsize=10) # pylint: map built-in referenced when not iterating (map-builtin-not-iterating) items = uploadpool.imap_unordered(lambda fi: fi.upload_to_storage(store), items, maxsize=10) - items = map(FetchItem.cleanup, items) + items = list(map(FetchItem.cleanup, items)) items = update_db_status(items) items = count_result_types(items, forprefix=forprefix) return ilen(items) # consume the generator diff --git a/idigbio_ingestion/mediaing/migrate.py b/idigbio_ingestion/mediaing/migrate.py index 990274fc..b55530a0 100644 --- a/idigbio_ingestion/mediaing/migrate.py +++ b/idigbio_ingestion/mediaing/migrate.py @@ -1,4 +1,4 @@ -from __future__ import division, absolute_import, print_function + from datetime import datetime diff --git a/idigbio_ingestion/mediaing/updatedb.py b/idigbio_ingestion/mediaing/updatedb.py index 201f491d..3f317de9 100644 --- a/idigbio_ingestion/mediaing/updatedb.py +++ b/idigbio_ingestion/mediaing/updatedb.py @@ -1,4 +1,4 @@ -from __future__ import division, absolute_import, print_function + from psycopg2.extensions import cursor @@ -108,7 +108,7 @@ def write_urls_to_db(to_insert, to_update): with apidbpool.cursor(autocommit=True) as cur: cur.executemany( "INSERT INTO media (url,type,mime) VALUES (%s,%s,%s)", - ((k, v[0], v[1]) for k,v in to_insert.items())) + ((k, v[0], v[1]) for k,v in list(to_insert.items()))) inserted = cur.rowcount cur.executemany( "UPDATE media SET type=%s, mime=%s, last_status=NULL, last_check=NULL WHERE url=%s", diff --git a/idigbio_ingestion/recordset_cleanup.py b/idigbio_ingestion/recordset_cleanup.py index 1264262a..d6eace8e 100644 --- a/idigbio_ingestion/recordset_cleanup.py +++ b/idigbio_ingestion/recordset_cleanup.py @@ -1,4 +1,4 @@ -from __future__ import division, absolute_import, print_function + import sys from idb.postgres_backend import apidbpool diff --git a/idigbio_ingestion/verify_ceph_objects.py b/idigbio_ingestion/verify_ceph_objects.py index 472f2937..45ed121e 100644 --- a/idigbio_ingestion/verify_ceph_objects.py +++ b/idigbio_ingestion/verify_ceph_objects.py @@ -3,8 +3,8 @@ https://www.idigbio.org/redmine/issues/1863 """ -from __future__ import division, absolute_import -from __future__ import print_function + + import io import logging From f901b0ad9080cc590f86ed2ddbbf7bcac2716594 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 27 Mar 2023 20:29:54 -0400 Subject: [PATCH 156/175] update local postgres test db version to 14 --- tests/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/README.md b/tests/README.md index 41da93d2..bdf30e9b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -123,7 +123,7 @@ Note: The data in the db with that name will be destroyed during testing. A temporary instance of postgres running in docker will suffice: ``` -$ docker run --rm --name postgres_test_idigbio --network host -e POSTGRES_PASSWORD=test -e POSTGRES_USER=test -e POSTGRES_DB=test_idigbio -d postgres:11 +$ docker run --rm --name postgres_test_idigbio --network host -e POSTGRES_PASSWORD=test -e POSTGRES_USER=test -e POSTGRES_DB=test_idigbio -d postgres:14 ``` ### WIP: run Elasticsearch in local docker the same way we run postgres @@ -158,7 +158,7 @@ Due to the dependencies mentioned above, you may wish to run the database in doc docker run --rm --name postgres_test_idigbio --network host \ -e POSTGRES_PASSWORD=test -e POSTGRES_USER=test -e POSTGRES_DB=test_idigbio \ - -d postgres:9.5 && \ + -d postgres:14 && \ sleep 5; \ py.test ; \ docker stop postgres_test_idigbio From 4242d2fa102d76e0110d44e85b16c6d538027b69 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 27 Mar 2023 20:32:36 -0400 Subject: [PATCH 157/175] use newer Pillow to remove deprecation warning --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4a31b992..59677642 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,7 @@ def read(*paths): extras_require={ 'ingestion': [ 'pydub==0.16.5', - pillow_package, + 'Pillow==6.1.0', 'lxml', 'chardet~=3.0', 'pyquery>=1.2', From 4983957ba9bc753b7a5eb1b32ab9cb9288cf6fb8 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 27 Mar 2023 21:03:44 -0400 Subject: [PATCH 158/175] attempt to fix Travis build issue Travis pipeline is failing with the following error: E ValueError: greenlet.greenlet has the wrong size, try recompiling. Expected 40, got 128 Possible fixes: https://github.com/gevent/gevent/issues/1260 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 59677642..23890a93 100644 --- a/setup.py +++ b/setup.py @@ -96,6 +96,7 @@ def read(*paths): 'pytest-flask~=0.15', 'pytest-mock==1.13.0', 'fakeredis', + 'greenlet==2.0.2' ] }, tests_require=['idb-backend[test]'], From 4408cbd525719ce4100d800127dabe9631e20bbd Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 27 Mar 2023 21:42:43 -0400 Subject: [PATCH 159/175] eliminate "greenlet has the wrong size" error during docker build --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 23890a93..126e7b16 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,8 @@ def read(*paths): 'coverage', 'numpy', 'scipy', - 'gevent==1.3.0', + 'gevent==1.3.4', + 'greenlet==0.4.13', 'gipc>=0.6.0, <0.7.0', 'unicodecsv>=0.14.1, < 0.15.0', 'shapely', @@ -96,7 +97,6 @@ def read(*paths): 'pytest-flask~=0.15', 'pytest-mock==1.13.0', 'fakeredis', - 'greenlet==2.0.2' ] }, tests_require=['idb-backend[test]'], From c695c3459162aad620e2997fd9390549c119b331 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 27 Mar 2023 21:43:33 -0400 Subject: [PATCH 160/175] update pip in Dockerfile build --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 30475820..764abd63 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,5 +17,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY . /opt/idb-backend/ WORKDIR /opt/idb-backend/ +RUN pip install --upgrade pip RUN pip --no-cache-dir install -e .[test] USER www-data From 3898838d6a3884f9723ac859b5265f7708cbaf19 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 27 Mar 2023 22:09:51 -0400 Subject: [PATCH 161/175] bump gevent to fix pgpool hang --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 126e7b16..67972966 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ def read(*paths): 'coverage', 'numpy', 'scipy', - 'gevent==1.3.4', + 'gevent==1.3.5', 'greenlet==0.4.13', 'gipc>=0.6.0, <0.7.0', 'unicodecsv>=0.14.1, < 0.15.0', From baa50f698057e2afb136d3cb0177a27e0d7a10ad Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 27 Mar 2023 22:17:45 -0400 Subject: [PATCH 162/175] add travis_wait to prevent pipeline timeouts --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a3c03e39..c4ce70ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ git: depth: 1 install: -- docker build -t $BUILDING . +- travis_wait 20 docker build -t $BUILDING . - psql -U postgres -c 'create database test_idigbio;' - psql -U postgres -c "DROP SCHEMA public CASCADE;" test_idigbio - docker run -d --net host --name minio-test -e "MINIO_ACCESS_KEY=${ACCESS_KEY}" -e "MINIO_SECRET_KEY=${SECRET_KEY=}" minio/minio:RELEASE.2017-07-24T18-27-35Z server /export From 0097eef88bc0f60fa75b657a65e6b45e85b253b7 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 27 Mar 2023 22:28:03 -0400 Subject: [PATCH 163/175] bump version to try to fix pgpool hang in Travis --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 67972966..de95be35 100644 --- a/setup.py +++ b/setup.py @@ -62,8 +62,8 @@ def read(*paths): 'coverage', 'numpy', 'scipy', - 'gevent==1.3.5', - 'greenlet==0.4.13', + 'gevent==1.3.6', + 'greenlet==0.4.14', 'gipc>=0.6.0, <0.7.0', 'unicodecsv>=0.14.1, < 0.15.0', 'shapely', From bc7d84b91ed162ebb76b618e722493815640a472 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Mon, 27 Mar 2023 22:35:12 -0400 Subject: [PATCH 164/175] change gevent and greenlet version pins --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index de95be35..dd59a783 100644 --- a/setup.py +++ b/setup.py @@ -62,8 +62,8 @@ def read(*paths): 'coverage', 'numpy', 'scipy', - 'gevent==1.3.6', - 'greenlet==0.4.14', + 'gevent>=1.3.6', + 'greenlet>=0.4.14', 'gipc>=0.6.0, <0.7.0', 'unicodecsv>=0.14.1, < 0.15.0', 'shapely', From 4b87bd8af99ede2c17d219c6df297833088446ac Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 28 Mar 2023 19:36:38 -0400 Subject: [PATCH 165/175] bump postgres version in travis pipeline --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c4ce70ba..a221c497 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ services: - docker - postgresql addons: - postgresql: "9.5" + postgresql: "14" git: depth: 1 From 94fd58994e97bb67e6fbc66253973efb16b7f89c Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 28 Mar 2023 19:43:58 -0400 Subject: [PATCH 166/175] use addon packages, port, and pguser as recommended in docs --- .travis.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.travis.yml b/.travis.yml index a221c497..5b3d0f21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,15 @@ services: - postgresql addons: postgresql: "14" + apt: + packages: + - postgresql-14 + - postgresql-client-14 +env: + global: + - PGPORT=5433 + - PGUSER=travis + git: depth: 1 From 27c1e40bd4ee5f5c22ab310cf768680a2cc5d715 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 28 Mar 2023 19:49:41 -0400 Subject: [PATCH 167/175] change to newer dist --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5b3d0f21..a24d6bf2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,10 @@ sudo: required -dist: xenial +dist: focal language: python python: 3.8.10 services: - docker -- postgresql +#- postgresql addons: postgresql: "14" apt: From f60397c65a32584919165aa6c8d3fa4c913412e7 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 28 Mar 2023 19:55:11 -0400 Subject: [PATCH 168/175] try to install pg from official repos --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index a24d6bf2..bbebc11a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,10 @@ env: - PGPORT=5433 - PGUSER=travis +before_install: + - apt install postgresql-common + - sh /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh + git: depth: 1 From cff94cb43eb721d5fe9e6107a7c3b9a17cdc411c Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 28 Mar 2023 19:59:58 -0400 Subject: [PATCH 169/175] try pg 11 instead --- .travis.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index bbebc11a..ae1af2e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,22 +4,17 @@ language: python python: 3.8.10 services: - docker -#- postgresql addons: - postgresql: "14" + postgresql: "11" apt: packages: - - postgresql-14 - - postgresql-client-14 + - postgresql-11 + - postgresql-client-11 env: global: - PGPORT=5433 - PGUSER=travis -before_install: - - apt install postgresql-common - - sh /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh - git: depth: 1 @@ -35,7 +30,7 @@ script: -e "IDB_STORAGE_SECRET_KEY=${SECRET_KEY}" -e "IDB_STORAGE_HOST=localhost:9000" $BUILDING - pytest -p no:cacheprovider --pguser=postgres --pgpass="" tests/idb + pytest -p no:cacheprovider --pguser=travis --pgpass="" tests/idb after_success: # upload new docker images From 64d9a07e0103b62fe28f52573a0c7a4106629ea6 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 28 Mar 2023 20:02:14 -0400 Subject: [PATCH 170/175] try pg 12 instead --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ae1af2e2..68e87eac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,11 +5,11 @@ python: 3.8.10 services: - docker addons: - postgresql: "11" + postgresql: "12" apt: packages: - - postgresql-11 - - postgresql-client-11 + - postgresql-12 + - postgresql-client-12 env: global: - PGPORT=5433 From 85f592e28bbd4742517d0abacb6c7dfb76d9fdf4 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 28 Mar 2023 20:08:54 -0400 Subject: [PATCH 171/175] try jammy to get access to pg 14, move envs --- .travis.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 68e87eac..b8205d14 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,19 +1,15 @@ sudo: required -dist: focal +dist: jammy language: python python: 3.8.10 services: - docker addons: - postgresql: "12" + postgresql: "14" apt: packages: - - postgresql-12 - - postgresql-client-12 -env: - global: - - PGPORT=5433 - - PGUSER=travis + - postgresql-14 + - postgresql-client-14 git: depth: 1 @@ -50,6 +46,8 @@ env: - BUILDING=$DOCKER_IMAGE:$TRAVIS_COMMIT - ACCESS_KEY="testAccessKey" - SECRET_KEY="testSecretKey" + - PGPORT=5433 + - PGUSER=travis notifications: slack: secure: "GVJcqmhKww+74MaAKiSPIf2oPxEFp0PoYHzdzL0xw5xd62OiJkrcdzOsiZ5DQxLlgU+MgWrxcxJmunMXOUvR7gt1V1WjuvYsmJ6+bC4/AU81Qr0z5BDANZRX7O8yA/WLvPyE5hsJQWAyBL0NFvkKbxzw/qKlaaoh+uUXI2FLA+LMq6kuW1BYyQuFsoX/14842ppRETrxK5uVAft7c+Egh68Tet5xO5lry3bBFZ0F75Jg5nuzgAgq3OcOe8Luo8FuekX75tBjT0hBNSM+Kn9LOPx25O6tRfivCJn1quOPNSNhet8zbeYecnSevW49VLtWi34fLX6rIsPcvGZg1Z5q1MjvxqDtuIEnhLBeR/8+hK+dLu6MP0YGcG4jBELEH13WlkmJJcH1X+DmRLK1tkgwWhJlW1RJ/QHPoqsMKMNFxS4aDenFGiYXSvv7skX8pr+i2QW1j55V6AU55OCvhsrgyHwFXlA+1ItU5geCVvmLscg8RXF7HuTrZrN/UxLjDVPAjY4rHS2ccZx1g3vo+K0OWafKk6yOi9J7rfwT+OGCMGv3Q57i1ShDIHvmfzfJmsjhtRCvKowwUrw2/Udean/ncdcRLvTTv0Xn+ioxnWAyvWo7UMdXj7YeHvp4bT9pY2eSjfHjB6Fe7S3vWC5zzLc0BDYouxkysv+j0EvP1vx84t4=" From 55207f6b0f6aa94cefcf5295abd94652ece49f59 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 28 Mar 2023 20:11:42 -0400 Subject: [PATCH 172/175] back to focal and pg 12 --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index b8205d14..19fb270d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,15 @@ sudo: required -dist: jammy +dist: focal language: python python: 3.8.10 services: - docker addons: - postgresql: "14" + postgresql: "12" apt: packages: - - postgresql-14 - - postgresql-client-14 + - postgresql-12 + - postgresql-client-12 git: depth: 1 From af2b3ea9fab03432cd1cde4b30ffa6b71f9660e2 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 28 Mar 2023 20:28:22 -0400 Subject: [PATCH 173/175] add output to see where jobs are aborting --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 19fb270d..5f55642e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ git: install: - travis_wait 20 docker build -t $BUILDING . +- echo "docker build complete!" - psql -U postgres -c 'create database test_idigbio;' - psql -U postgres -c "DROP SCHEMA public CASCADE;" test_idigbio - docker run -d --net host --name minio-test -e "MINIO_ACCESS_KEY=${ACCESS_KEY}" -e "MINIO_SECRET_KEY=${SECRET_KEY=}" minio/minio:RELEASE.2017-07-24T18-27-35Z server /export From c2e36c308845c75b6f7b2166f42275124e2647b0 Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 28 Mar 2023 20:42:34 -0400 Subject: [PATCH 174/175] reduce output to see if can avoid travis terminations --- Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 764abd63..938db1b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.8.10 # python:3.6 and later are based on "Debian GNU/Linux" -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN apt-get update > /dev/null && apt-get install -y --no-install-recommends \ gfortran \ libatlas-base-dev \ ffmpeg \ @@ -13,10 +13,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # fonts-dejavu-core \ # libxml2 \ # libxslt1-dev \ - && rm -rf /var/lib/apt/lists/* + > /dev/null && rm -rf /var/lib/apt/lists/* COPY . /opt/idb-backend/ WORKDIR /opt/idb-backend/ -RUN pip install --upgrade pip -RUN pip --no-cache-dir install -e .[test] +# RUN python3 -m virtualenv venv +RUN pip install -q --upgrade pip +RUN pip -q --no-cache-dir install -e .[test] USER www-data From ec6cacfee06bec6ae41e1f94265b36e732fb038d Mon Sep 17 00:00:00 2001 From: Dan Stoner Date: Tue, 28 Mar 2023 20:49:54 -0400 Subject: [PATCH 175/175] remove travis_wait and use "travis" for pg connection travis_wait does not work with docker build as argument (triggers abort). use travis db account when connecting diff --git a/.travis.yml b/.travis.yml index 5f55642..3e5fc60 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,10 +15,10 @@ git: depth: 1 install: -- travis_wait 20 docker build -t $BUILDING . +- docker build -t $BUILDING . - echo "docker build complete!" -- psql -U postgres -c 'create database test_idigbio;' -- psql -U postgres -c "DROP SCHEMA public CASCADE;" test_idigbio +- psql -U travis -c 'create database test_idigbio;' +- psql -U travis -c "DROP SCHEMA public CASCADE;" test_idigbio - docker run -d --net host --name minio-test -e "MINIO_ACCESS_KEY=${ACCESS_KEY}" -e "MINIO_SECRET_KEY=${SECRET_KEY=}" minio/minio:RELEASE.2017-07-24T18-27-35Z server /export script: --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5f55642e..3e5fc60b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,10 +15,10 @@ git: depth: 1 install: -- travis_wait 20 docker build -t $BUILDING . +- docker build -t $BUILDING . - echo "docker build complete!" -- psql -U postgres -c 'create database test_idigbio;' -- psql -U postgres -c "DROP SCHEMA public CASCADE;" test_idigbio +- psql -U travis -c 'create database test_idigbio;' +- psql -U travis -c "DROP SCHEMA public CASCADE;" test_idigbio - docker run -d --net host --name minio-test -e "MINIO_ACCESS_KEY=${ACCESS_KEY}" -e "MINIO_SECRET_KEY=${SECRET_KEY=}" minio/minio:RELEASE.2017-07-24T18-27-35Z server /export script: