From c421fd946d0463b667e1362334275b9ed4d1d4f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 19 Jul 2017 10:10:31 +0200 Subject: [PATCH 01/15] Bump version --- landsat/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/landsat/__init__.py b/landsat/__init__.py index deea98b..b5e195e 100644 --- a/landsat/__init__.py +++ b/landsat/__init__.py @@ -1 +1 @@ -__version__ = '0.13.1' +__version__ = '0.14.dev0' From d3a6f5d640d55d2a145d5ca13fff6e3f9755a20a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 19 Jul 2017 10:56:48 +0200 Subject: [PATCH 02/15] Loosen up version requirements of all dependencies Tested in: * Python 3.5 and 3.6 conda environments with latest NumPy and GDAL * Official Python 3.4, 3.5 and 3.6 Docker images based on Debian Jessie using only pip --- requirements.txt | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7a40899..de31a82 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,14 @@ -usgs==0.1.9 -requests==2.7.0 -python-dateutil==2.5.1 -numpy==1.10.4 -termcolor==1.1.0 -rasterio==0.32.0 +usgs>=0.1.9 +requests>=2.7.0 +python-dateutil>=2.5.1 +numpy>=1.10.4 +termcolor>=1.1.0 +rasterio>=0.32.0,<1* six>=1.8.0 -scipy==0.17.0 -scikit-image==0.12.3 -homura==0.1.3 -boto==2.39.0 -polyline==1.3 -geocoder==1.9.0 -matplotlib==1.5.1 +scipy>=0.17.0 +scikit-image>=0.12.3 +homura>=0.1.3 +boto>=2.39.0 +polyline>=1.3 +geocoder>=1.9.0 +matplotlib>=1.5.1 From 3ac5c771d0d0c53cd0cbf30ba07939cdc22167a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 19 Jul 2017 11:54:58 +0200 Subject: [PATCH 03/15] Modernize Dockerfile, fix #226 --- Dockerfile | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index b42bd00..f393901 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,9 @@ FROM ubuntu:14.04 RUN apt-get -y update -RUN apt-get install --yes git-core python-pip python-scipy libgdal-dev libatlas-base-dev gfortran libfreetype6-dev libglib2.0-dev zlib1g-dev python-pycurl +RUN apt-get install --yes curl libgdal-dev python3-dev build-essential ADD . /landsat -RUN pip install setuptools -RUN pip install -U pip -RUN pip install wheel -RUN pip install https://s3-us-west-2.amazonaws.com/ds-satellite-projects/landsat-util/numpy-1.10.4-cp27-cp27mu-linux_x86_64.whl -RUN pip install https://s3-us-west-2.amazonaws.com/ds-satellite-projects/landsat-util/Pillow-3.1.1-cp27-cp27mu-linux_x86_64.whl -RUN pip install https://s3-us-west-2.amazonaws.com/ds-satellite-projects/landsat-util/scikit_image-0.12.3-cp27-cp27mu-manylinux1_x86_64.whl +RUN curl -O https://bootstrap.pypa.io/get-pip.py +RUN python3 get-pip.py +RUN pip3 install numpy RUN cd /landsat && pip install -r requirements-dev.txt -RUN sed -i 's/numpy.*//g' /landsat/requirements.txt -RUN sed -i 's/scipy.*//g' /landsat/requirements.txt -RUN sed -i 's/scikit-image.*//g' /landsat/requirements.txt RUN cd /landsat && pip install -e . From 671d72e71f64cc0926a53e014d2b9f279ebce030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 19 Jul 2017 12:27:23 +0200 Subject: [PATCH 04/15] Update installation docs --- docs/installation.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 302f906..65224b3 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -11,11 +11,18 @@ Mac OSX Ubuntu 14.04 ++++++++++++ -Use pip to install landsat-util. If you are not using virtualenv, you might have to run ``pip`` as ``sudo``:: +Use pip to install landsat-util. Follow these instructions to `install pip on Ubuntu`_:: $: sudo apt-get update - $: sudo apt-get install python-pip python-numpy python-scipy libgdal-dev libatlas-base-dev gfortran libfreetype6-dev - $: pip install landsat-util + $: sudo apt-get install libgdal-dev python3-dev build-essential + $: pip3 install numpy + $: pip3 install landsat-util + +.. _`install pip on Ubuntu`: https://pip.pypa.io/en/stable/installing/#installing-with-get-pip-py + +If you are not using virtualenv, you might have to use the ``--user`` flag with ``pip`` (see `description of pip user installs`_). + +.. _`description of pip user installs`: https://pip.pypa.io/en/stable/user_guide/#user-installs Other systems +++++++++++++ From 12779aad2fe9b589de0d68395042542f685f862b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Fri, 21 Jul 2017 12:18:05 +0200 Subject: [PATCH 05/15] Refine version requirements with upper bounds --- requirements.txt | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/requirements.txt b/requirements.txt index de31a82..96e701e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,14 @@ -usgs>=0.1.9 -requests>=2.7.0 -python-dateutil>=2.5.1 -numpy>=1.10.4 -termcolor>=1.1.0 -rasterio>=0.32.0,<1* -six>=1.8.0 -scipy>=0.17.0 -scikit-image>=0.12.3 -homura>=0.1.3 -boto>=2.39.0 -polyline>=1.3 -geocoder>=1.9.0 -matplotlib>=1.5.1 +usgs>=0.1.9,<1.* +requests~=2.7 +python-dateutil>=2.5.1,<3.* +numpy>=1.10.4,<2.* +termcolor~=1.1 +rasterio~=0.32 +six~=1.8 +scipy~=0.17 +scikit-image>=0.12.3,<1.* +homura>=0.1.3,<1.* +boto~=2.39 +polyline~=1.3 +geocoder~=1.9 +matplotlib>=1.5.1,<2.* From 84d5f4bbb419f0a155145e47b9d33b5e6676c052 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Tue, 12 Jun 2018 23:29:11 -0400 Subject: [PATCH 06/15] update cloudCoverFull to cloud_coverage (resolves #249) --- landsat/search.py | 6 +++--- tests/test_search.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/landsat/search.py b/landsat/search.py index a7d956e..2c32710 100644 --- a/landsat/search.py +++ b/landsat/search.py @@ -118,7 +118,7 @@ def search(self, paths_rows=None, lat=None, lon=None, address=None, start_date=N 'path': three_digit(r['path']), 'thumbnail': r['browseURL'], 'date': r['acquisitionDate'], - 'cloud': r['cloudCoverFull'] + 'cloud': r['cloud_coverage'] }, 'geometry': { 'type': 'Polygon', @@ -147,7 +147,7 @@ def search(self, paths_rows=None, lat=None, lon=None, address=None, start_date=N 'row': three_digit(i['row']), 'thumbnail': i['browseURL'], 'date': i['acquisitionDate'], - 'cloud': i['cloudCoverFull']} + 'cloud': i['cloud_coverage']} for i in r_dict['results']] return result @@ -288,7 +288,7 @@ def cloud_cover_prct_range_builder(self, min=0, max=100): :returns: String """ - return 'cloudCoverFull:[%s+TO+%s]' % (min, max) + return 'cloud_coverage:[%s+TO+%s]' % (min, max) def address_builder(self, address): """ Builds lat and lon query from a geocoded address. diff --git a/tests/test_search.py b/tests/test_search.py index 792da6d..11a5009 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -75,7 +75,7 @@ def test_query_builder(self): self.assertRaises(ValueError, self.s.query_builder, paths_rows='003,004,010') # full example - expected_string = ('acquisitionDate:[2014-01-01+TO+2014-11-12]+AND+cloudCoverFull:[10+TO+28]+AND+upperLeftCo' + expected_string = ('acquisitionDate:[2014-01-01+TO+2014-11-12]+AND+cloud_coverage:[10+TO+28]+AND+upperLeftCo' 'rnerLatitude:[23+TO+1000]+AND+lowerRightCornerLatitude:[-1000+TO+23]+AND+lowerLeftCorner' 'Longitude:[-1000+TO+21]+AND+upperRightCornerLongitude:[21+TO+1000]+AND+((path:003+AND+ro' 'w:004))') @@ -99,19 +99,19 @@ def test_lat_lon_builder(self): def test_cloud_cover_prct_range_builder(self): # no input string = self.s.cloud_cover_prct_range_builder() - self.assertEqual('cloudCoverFull:[0+TO+100]', string) + self.assertEqual('cloud_coverage:[0+TO+100]', string) # just min string = self.s.cloud_cover_prct_range_builder(3) - self.assertEqual('cloudCoverFull:[3+TO+100]', string) + self.assertEqual('cloud_coverage:[3+TO+100]', string) # just max string = self.s.cloud_cover_prct_range_builder(max=30) - self.assertEqual('cloudCoverFull:[0+TO+30]', string) + self.assertEqual('cloud_coverage:[0+TO+30]', string) # both inputs string = self.s.cloud_cover_prct_range_builder(7, 10) - self.assertEqual('cloudCoverFull:[7+TO+10]', string) + self.assertEqual('cloud_coverage:[7+TO+10]', string) def test_date_range_builder(self): string = self.s.date_range_builder('2014-01-01', '2015-01-01') From 135238c11de18e49385cd18efecd6cccc3b69007 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Tue, 12 Jun 2018 23:45:48 -0400 Subject: [PATCH 07/15] remove unused pycurl import --- landsat/landsat.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/landsat/landsat.py b/landsat/landsat.py index af799cc..ef04475 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -18,7 +18,6 @@ from datetime import datetime from dateutil.relativedelta import relativedelta from dateutil.parser import parse -import pycurl from boto.exception import NoAuthHandlerFound from .downloader import Downloader, IncorrectSceneId, RemoteFileDoesntExist, USGSInventoryAccessMissing @@ -493,5 +492,5 @@ def __main__(): if __name__ == "__main__": try: __main__() - except (KeyboardInterrupt, pycurl.error): + except KeyboardInterrupt: exit('Received Ctrl + C... Exiting! Bye.', 1) From 0dbf5ce619c8817c373784f30b930a478b41417a Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Wed, 13 Jun 2018 00:00:16 -0400 Subject: [PATCH 08/15] switch to pytest from nose --- requirements-dev.txt | 2 +- setup.py | 1 - tox.ini | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 7f71cc4..145dc3a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ pdoc>=0.3.1 -nose>=1.3.7 coverage>=4.0 Sphinx>=1.3.1 wheel>=0.26.0 mock>=1.3.0 jsonschema==2.5.1 +pytest==3.6.1 \ No newline at end of file diff --git a/setup.py b/setup.py index b0a19fc..b11104f 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,6 @@ def readme(): license='CCO', platforms='Posix; MacOS X; Windows', install_requires=INSTALL_REQUIRES, - test_suite='nose.collector', tests_require=TEST_REQUIRES, **setup_kwargs ) diff --git a/tox.ini b/tox.ini index 426378a..1f8e3c1 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,6 @@ deps = pip>=8.1.1 jsonschema mock>=1.3.0 - nose>=1.3.7 pytest commands = pip install -r requirements.txt From c28b703fcb7c8d4e193778e912ee86899370a089 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Wed, 13 Jun 2018 00:24:22 -0400 Subject: [PATCH 09/15] remove unneeded file --- landsat-util.sublime-project | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 landsat-util.sublime-project diff --git a/landsat-util.sublime-project b/landsat-util.sublime-project deleted file mode 100644 index 1ca47d9..0000000 --- a/landsat-util.sublime-project +++ /dev/null @@ -1,22 +0,0 @@ -{ - "build_systems": - [ - { - "file_regex": "^[ ]*File \"(...*?)\", line ([0-9]*)", - "name": "Anaconda Python Builder", - "selector": "source.python", - "shell_cmd": "~/.virtualenvs/landsat-util/bin/python -u \"$file\"" - } - ], - "folders": - [ - { - "follow_symlinks": true, - "path": "." - } - ], - "settings": - { - "python_interpreter": "~/.virtualenvs/landsat-util/bin/python" - } -} From b546c761e446664ebc16bc17a0077a0b57c6269b Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Mon, 30 Jul 2018 16:23:40 -0400 Subject: [PATCH 10/15] update Dockerfile, add docker-compose --- Dockerfile | 30 +++++++++++++++++++++--------- docker-compose.yml | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 9 deletions(-) create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile index f393901..82eccbe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,21 @@ -FROM ubuntu:14.04 -RUN apt-get -y update -RUN apt-get install --yes curl libgdal-dev python3-dev build-essential -ADD . /landsat -RUN curl -O https://bootstrap.pypa.io/get-pip.py -RUN python3 get-pip.py -RUN pip3 install numpy -RUN cd /landsat && pip install -r requirements-dev.txt -RUN cd /landsat && pip install -e . +FROM developmentseed/geolambda:full + +#RUN \ +# yum install -y openssl-devel + +ENV \ + PYCURL_SSL_LIBRARY=nss + +WORKDIR /build + +COPY requirements*txt /build/ + +RUN \ + pip-3.6 install -r requirements.txt; \ + pip-3.6 install -r requirements-dev.txt; + +COPY . /build/ + +RUN pip-3.6 install . + +WORKDIR /home/geolambda diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..92cbd63 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,37 @@ +version: '2' + +services: + + base: + build: + context: . + image: 'developmentseed/landsat-util:latest' + entrypoint: /bin/bash + env_file: .env + volumes: + - '.:/home/geolambda/work' + + test: + image: 'developmentseed/landsat-util:latest' + working_dir: /home/geolambda + entrypoint: bash -c 'pytest tests/'; + env_file: .env + volumes: + - './lambda:/home/geolambda/lambda' + - './tests:/home/geolambda/tests' + + package: + image: 'developmentseed/landsat-util:latest' + command: lambda-package.sh + working_dir: /home/geolambda/work + volumes: + - '.:/home/geolambda/work' + + testpackage: + image: 'developmentseed/geolambda:base' + command: bash -c "pytest tests/"; + working_dir: /home/geolambda + env_file: .env + volumes: + - './tests:/home/geolambda/tests' + From 0477ee2a7556b32da2015ee85af5229a23b77354 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Mon, 30 Jul 2018 16:23:51 -0400 Subject: [PATCH 11/15] update rasterio version, remove matplotlib --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 96e701e..e7a5104 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ requests~=2.7 python-dateutil>=2.5.1,<3.* numpy>=1.10.4,<2.* termcolor~=1.1 -rasterio~=0.32 +rasterio~=0.36 six~=1.8 scipy~=0.17 scikit-image>=0.12.3,<1.* @@ -11,4 +11,3 @@ homura>=0.1.3,<1.* boto~=2.39 polyline~=1.3 geocoder~=1.9 -matplotlib>=1.5.1,<2.* From 331a380fbfc1fc715c77a145fb23bf4cfe0346cd Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Mon, 30 Jul 2018 16:47:33 -0400 Subject: [PATCH 12/15] update usgs lib to 0.2.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e7a5104..d77dab2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -usgs>=0.1.9,<1.* +usgs>=0.2.1 requests~=2.7 python-dateutil>=2.5.1,<3.* numpy>=1.10.4,<2.* From 2579d2280a08bc6456e968f12de658c9926e1bdd Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Mon, 30 Jul 2018 17:01:27 -0400 Subject: [PATCH 13/15] add name to AUTHORS --- AUTHORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.txt b/AUTHORS.txt index 8bb02d1..1f381d7 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -6,5 +6,6 @@ Marc Farra https://github.com/kamicut Drew Bollinger https://github.com/drewbo Sean Gillies https://github.com/sgillies Alex Kappel https://twitter.com/alex_kappel +Matthew Hanson https://github.com/matthewhanson See also https://github.com/developmentseed/landsat-util/graphs/contributors. From 6110ff7d59ed08429deb0f2b9ab9748641bdf0cb Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Mon, 30 Jul 2018 17:03:51 -0400 Subject: [PATCH 14/15] add docker-compose task for running landsat CLI --- docker-compose.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 92cbd63..8243c3f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,14 @@ services: volumes: - '.:/home/geolambda/work' + landsat: + image: 'developmentseed/landsat-util:latest' + working_dir: /home/geolambda + entrypoint: landsat + env_file: .env + volumes: + - '.:/home/geolambda/work' + test: image: 'developmentseed/landsat-util:latest' working_dir: /home/geolambda From 9e30367aca66aa0539d2c9eefb68db59b47f4c85 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Mon, 30 Jul 2018 17:08:39 -0400 Subject: [PATCH 15/15] bump version to 0.14.0 --- landsat/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/landsat/__init__.py b/landsat/__init__.py index b5e195e..ef91994 100644 --- a/landsat/__init__.py +++ b/landsat/__init__.py @@ -1 +1 @@ -__version__ = '0.14.dev0' +__version__ = '0.14.0'