Skip to content

Commit 981c368

Browse files
author
Ayaz Salikhov
committed
Merge branch 'master' into asalikhov/remove_py2
2 parents 7b43439 + 95ccda3 commit 981c368

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

93 files changed

+3144
-2141
lines changed

‎.flake8

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
[flake8]
2+
ignore = W605,W503,W504,H306,H238,H301,H202
3+
max-line-length = 120
4+
per-file-ignores =
5+
test/test_packages.py:E501

‎.github/workflows/docker.yml

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
name: Build Docker Images
2+
3+
on:
4+
pull_request:
5+
paths-ignore:
6+
- "*.md"
7+
- "binder/**"
8+
- "docs/**"
9+
- "examples/**"
10+
push:
11+
branches:
12+
- master
13+
paths-ignore:
14+
- "*.md"
15+
- "binder/**"
16+
- "docs/**"
17+
- "examples/**"
18+
19+
jobs:
20+
build:
21+
name: Build Docker Images
22+
runs-on: ubuntu-latest
23+
if: >
24+
!contains(github.event.head_commit.message, 'ci skip') &&
25+
!contains(github.event.pull_request.title, 'ci skip')
26+
steps:
27+
- name: Clone Main Repo
28+
uses: actions/checkout@v2
29+
with:
30+
path: main
31+
- name: Clone Wiki
32+
uses: actions/checkout@v2
33+
with:
34+
repository: ${{github.repository}}.wiki
35+
path: wiki
36+
- name: Set Up Python
37+
uses: actions/setup-python@v2
38+
with:
39+
python-version: 3.8
40+
- name: Install Dev Dependencies
41+
run: |
42+
python -m pip install --upgrade pip
43+
make -C main dev-env hadolint-install
44+
- name: Lint Dockerfiles
45+
run: make -C main hadolint-all
46+
- name: Run pre-commit hooks
47+
run: make -C main pre-commit-all
48+
- name: Build Docker Images
49+
run: make -C main build-test-all
50+
- name: Run Post-Build Hooks
51+
run: make -C main hook-all
52+
env:
53+
COMMIT_MSG: "${{github.event.head_commit.message}}"
54+
WIKI_PATH: ../wiki
55+
- name: Login to Docker Hub
56+
if: github.ref == 'refs/heads/master'
57+
run: >
58+
echo '${{secrets.DOCKERHUB_PASSWORD}}' | docker login --username
59+
'${{secrets.DOCKERHUB_USERNAME}}' --password-stdin
60+
- name: Push Images to DockerHub
61+
if: github.ref == 'refs/heads/master'
62+
run: make -C main push-all
63+
- name: Push Wiki to GitHub
64+
if: github.ref == 'refs/heads/master'
65+
# Pass GITHUB_REPOSITORY directly to avoid conflict with GitHub Actions built-in env var
66+
run: make -C main git-commit GITHUB_REPOSITORY='${{ github.repository }}.wiki'
67+
env:
68+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
69+
LOCAL_PATH: ../wiki

‎.github/workflows/sphinx.yml

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
name: Build Sphinx Documentation
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- "docs/**"
7+
- ".github/workflows/sphinx.yml"
8+
push:
9+
branches:
10+
- master
11+
paths:
12+
- "docs/**"
13+
- ".github/workflows/sphinx.yml"
14+
15+
jobs:
16+
build:
17+
name: Build Sphinx Documentation
18+
runs-on: ubuntu-latest
19+
if: >
20+
!contains(github.event.head_commit.message , 'ci skip') &&
21+
!contains(github.event.pull_request.title, 'ci skip')
22+
steps:
23+
- name: Checkout Repo
24+
uses: actions/checkout@v2
25+
- name: Set Up Python
26+
uses: actions/setup-python@v2
27+
with:
28+
python-version: 3.x
29+
- name: Install Dev Dependencies
30+
run: |
31+
python -m pip install --upgrade pip
32+
make dev-env
33+
- name: Build Documentation
34+
run: make docs
35+
- name: Extract Source Strings
36+
if: github.ref == 'refs/heads/master'
37+
working-directory: docs
38+
run: |
39+
make gettext
40+
sphinx-intl update -p _build/gettext -l en
41+
- name: Push Strings to Master
42+
if: github.ref == 'refs/heads/master'
43+
run: make git-commit
44+
env:
45+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
46+
GITHUB_REPOSITORY: ${{ github.repository }}
47+
LOCAL_PATH: ./docs/locale/en

‎.pre-commit-config.yaml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
---
2+
repos:
3+
- repo: https://github.com/pre-commit/pre-commit-hooks
4+
rev: v3.1.0
5+
hooks:
6+
- id: check-yaml
7+
files: .*\.(yaml|yml)$
8+
- repo: https://github.com/adrienverge/yamllint.git
9+
rev: v1.23.0
10+
hooks:
11+
- id: yamllint
12+
args: ['-d {extends: relaxed, rules: {line-length: disable}}', '-s']
13+
files: \.(yaml|yml)$
14+
- repo: https://github.com/openstack-dev/bashate.git
15+
rev: 2.0.0
16+
hooks:
17+
- id: bashate
18+
args: ['--ignore=E006']
19+
- repo: https://gitlab.com/pycqa/flake8
20+
rev: 3.8.3
21+
hooks:
22+
- id: flake8
23+
- repo: https://github.com/pre-commit/mirrors-autopep8
24+
rev: v1.5.4
25+
hooks:
26+
- id: autopep8

‎.travis.yml

Lines changed: 0 additions & 48 deletions
This file was deleted.

‎Makefile

Lines changed: 59 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,8 @@
44

55
# Use bash for inline if-statements in arch_patch target
66
SHELL:=bash
7-
OWNER:=jupyter
87
ARCH:=$(shell uname -m)
9-
DIFF_RANGE?=master...HEAD
8+
OWNER?=jupyter
109

1110
# Need to list the images in build dependency order
1211
ifeq ($(ARCH),ppc64le)
@@ -24,8 +23,9 @@ endif
2423

2524
ALL_IMAGES:=$(ALL_STACKS)
2625

27-
# Linter
26+
# Dockerfile Linter
2827
HADOLINT="${HOME}/hadolint"
28+
HADOLINT_VERSION="v1.18.0"
2929

3030
help:
3131
# http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
@@ -74,24 +74,38 @@ dev/%: ## run a foreground container for a stack
7474
docker run -it --rm -p $(PORT):8888 $(DARGS) $(OWNER)/$(notdir $@) $(ARGS)
7575

7676
dev-env: ## install libraries required to build docs and run tests
77-
pip install -r requirements-dev.txt
77+
@pip install -r requirements-dev.txt
7878

79-
lint/%: ARGS?=
80-
lint/%: ## lint the dockerfile(s) for a stack
81-
@echo "Linting Dockerfiles in $(notdir $@)..."
82-
@git ls-files --exclude='Dockerfile*' --ignored $(notdir $@) | grep -v ppc64 | xargs -L 1 $(HADOLINT) $(ARGS)
83-
@echo "Linting done!"
84-
85-
lint-all: $(foreach I,$(ALL_IMAGES),lint/$(I) ) ## lint all stacks
79+
docs: ## build HTML documentation
80+
make -C docs html
8681

87-
lint-build-test-all: $(foreach I,$(ALL_IMAGES),lint/$(I) arch_patch/$(I) build/$(I) test/$(I) ) ## lint, build and test all stacks
88-
89-
lint-install: ## install hadolint
90-
@echo "Installing hadolint at $(HADOLINT) ..."
91-
@curl -sL -o $(HADOLINT) "https://github.com/hadolint/hadolint/releases/download/v1.18.0/hadolint-$(shell uname -s)-$(shell uname -m)"
92-
@chmod 700 $(HADOLINT)
93-
@echo "Installation done!"
94-
@$(HADOLINT) --version
82+
git-commit: LOCAL_PATH?=.
83+
git-commit: GITHUB_SHA?=$(shell git rev-parse HEAD)
84+
git-commit: GITHUB_REPOSITORY?=jupyter/docker-stacks
85+
git-commit: GITHUB_TOKEN?=
86+
git-commit: ## commit outstading git changes and push to remote
87+
@git config --global user.name "GitHub Actions"
88+
@git config --global user.email "[email protected]"
89+
90+
@echo "Publishing outstanding changes in $(LOCAL_PATH) to $(GITHUB_REPOSITORY)"
91+
@cd $(LOCAL_PATH) && \
92+
git remote add publisher https://$(GITHUB_TOKEN)@github.com/$(GITHUB_REPOSITORY).git && \
93+
git checkout master && \
94+
git add -A -- . && \
95+
git commit -m "[ci skip] Automated publish for $(GITHUB_SHA)" || exit 0
96+
@cd $(LOCAL_PATH) && git push -u publisher master
97+
98+
hook/%: export COMMIT_MSG?=$(shell git log -1 --pretty=%B)
99+
hook/%: export GITHUB_SHA?=$(shell git rev-parse HEAD)
100+
hook/%: export WIKI_PATH?=../wiki
101+
hook/%: ## run post-build hooks for an image
102+
BUILD_TIMESTAMP="$$(date -u +%FT%TZ)" \
103+
DOCKER_REPO="$(OWNER)/$(notdir $@)" \
104+
IMAGE_NAME="$(OWNER)/$(notdir $@):latest" \
105+
IMAGE_SHORT_NAME="$(notdir $@)" \
106+
$(SHELL) $(notdir $@)/hooks/run_hook
107+
108+
hook-all: $(foreach I,$(ALL_IMAGES),hook/$(I) ) ## run post-build hooks for all images
95109

96110
img-clean: img-rm-dang img-rm ## clean dangling and jupyter images
97111

@@ -107,20 +121,40 @@ img-rm-dang: ## remove dangling images (tagged None)
107121
@echo "Removing dangling images ..."
108122
-docker rmi --force $(shell docker images -f "dangling=true" -q) 2> /dev/null
109123

110-
docs: ## build HTML documentation
111-
make -C docs html
124+
hadolint/%: ARGS?=
125+
hadolint/%: ## lint the dockerfile(s) for a stack
126+
@echo "Linting Dockerfiles in $(notdir $@)..."
127+
@git ls-files --exclude='Dockerfile*' --ignored $(notdir $@) | grep -v ppc64 | xargs -L 1 $(HADOLINT) $(ARGS)
128+
@echo "Linting done!"
129+
130+
hadolint-all: $(foreach I,$(ALL_IMAGES),hadolint/$(I) ) ## lint all stacks
131+
132+
hadolint-build-test-all: $(foreach I,$(ALL_IMAGES),hadolint/$(I) arch_patch/$(I) build/$(I) test/$(I) ) ## lint, build and test all stacks
112133

113-
n-docs-diff: ## number of docs/ files changed since branch from master
114-
@git diff --name-only $(DIFF_RANGE) -- docs/ ':!docs/locale' | wc -l | awk '{print $$1}'
134+
hadolint-install: ## install hadolint
135+
@echo "Installing hadolint at $(HADOLINT) ..."
136+
@curl -sL -o $(HADOLINT) "https://github.com/hadolint/hadolint/releases/download/$(HADOLINT_VERSION)/hadolint-$(shell uname -s)-$(shell uname -m)"
137+
@chmod 700 $(HADOLINT)
138+
@echo "Installation done!"
139+
@$(HADOLINT) --version
115140

141+
pre-commit-all: ## run pre-commit hook on all files
142+
@pre-commit run --all-files
116143

117-
n-other-diff: ## number of files outside docs/ changed since branch from master
118-
@git diff --name-only $(DIFF_RANGE) -- ':!docs/' | wc -l | awk '{print $$1}'
144+
pre-commit-install: ## set up the git hook scripts
145+
@pre-commit --version
146+
@pre-commit install
119147

120148
pull/%: DARGS?=
121149
pull/%: ## pull a jupyter image
122150
docker pull $(DARGS) $(OWNER)/$(notdir $@)
123151

152+
push/%: DARGS?=
153+
push/%: ## push all tags for a jupyter image
154+
docker push $(DARGS) $(OWNER)/$(notdir $@)
155+
156+
push-all: $(foreach I,$(ALL_IMAGES),push/$(I) ) ## push all tagged images
157+
124158
run/%: DARGS?=
125159
run/%: ## run a bash in interactive mode in a stack
126160
docker run -it --rm $(DARGS) $(OWNER)/$(notdir $@) $(SHELL)
@@ -129,20 +163,6 @@ run-sudo/%: DARGS?=
129163
run-sudo/%: ## run a bash in interactive mode as root in a stack
130164
docker run -it --rm -u root $(DARGS) $(OWNER)/$(notdir $@) $(SHELL)
131165

132-
tx-en: ## rebuild en locale strings and push to master (req: GH_TOKEN)
133-
@git config --global user.email "[email protected]"
134-
@git config --global user.name "Travis CI"
135-
@git checkout master
136-
137-
@make -C docs clean gettext
138-
@cd docs && sphinx-intl update -p _build/gettext -l en
139-
140-
@git add docs/locale/en
141-
@git commit -m "[ci skip] Update en source strings (build: $$TRAVIS_JOB_NUMBER)"
142-
143-
@git remote add origin-tx https://$${GH_TOKEN}@github.com/jupyter/docker-stacks.git
144-
@git push -u origin-tx master
145-
146166
test/%: ## run tests against a stack (only common tests or common tests + specific tests)
147167
@if [ ! -d "$(notdir $@)/test" ]; then TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -m "not info" test; \
148168
else TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -m "not info" test $(notdir $@)/test; fi

‎all-spark-notebook/Dockerfile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,17 @@ RUN apt-get update && \
1717
fonts-dejavu \
1818
gfortran \
1919
gcc && \
20-
rm -rf /var/lib/apt/lists/*
20+
apt-get clean && rm -rf /var/lib/apt/lists/*
2121

2222
USER $NB_UID
2323

2424
# R packages
2525
RUN conda install --quiet --yes \
26-
'r-base=3.6.3' \
26+
'r-base=4.0.3' \
2727
'r-ggplot2=3.3*' \
2828
'r-irkernel=1.1*' \
2929
'r-rcurl=1.98*' \
30-
'r-sparklyr=1.2*' \
30+
'r-sparklyr=1.4*' \
3131
&& \
3232
conda clean --all -f -y && \
3333
fix-permissions "${CONDA_DIR}" && \

‎all-spark-notebook/README.md

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,15 @@
1-
[![docker pulls](https://img.shields.io/docker/pulls/jupyter/all-spark-notebook.svg)](https://hub.docker.com/r/jupyter/all-spark-notebook/) [![docker stars](https://img.shields.io/docker/stars/jupyter/all-spark-notebook.svg)](https://hub.docker.com/r/jupyter/all-spark-notebook/) [![image metadata](https://images.microbadger.com/badges/image/jupyter/all-spark-notebook.svg)](https://microbadger.com/images/jupyter/all-spark-notebook "jupyter/all-spark-notebook image metadata")
1+
[![docker pulls](https://img.shields.io/docker/pulls/jupyter/all-spark-notebook.svg)](https://hub.docker.com/r/jupyter/all-spark-notebook/)
2+
[![docker stars](https://img.shields.io/docker/stars/jupyter/all-spark-notebook.svg)](https://hub.docker.com/r/jupyter/all-spark-notebook/)
3+
[![image metadata](https://images.microbadger.com/badges/image/jupyter/all-spark-notebook.svg)](https://microbadger.com/images/jupyter/all-spark-notebook "jupyter/all-spark-notebook image metadata")
24

35
# Jupyter Notebook Python, Scala, R, Spark Stack
46

5-
Please visit the documentation site for help using and contributing to this image and others.
7+
GitHub Actions in the https://github.com/jupyter/docker-stacks project builds and pushes this image
8+
to Docker Hub.
69

7-
* [Jupyter Docker Stacks on ReadTheDocs](http://jupyter-docker-stacks.readthedocs.io/en/latest/index.html)
8-
* [Selecting an Image :: Core Stacks :: jupyter/all-spark-notebook](http://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-all-spark-notebook)
9-
* [Image Specifics :: Apache Spark](http://jupyter-docker-stacks.readthedocs.io/en/latest/using/specifics.html#apache-spark)
10+
Please visit the project documentation site for help using and contributing to this image and
11+
others.
12+
13+
- [Jupyter Docker Stacks on ReadTheDocs](http://jupyter-docker-stacks.readthedocs.io/en/latest/index.html)
14+
- [Selecting an Image :: Core Stacks :: jupyter/all-spark-notebook](http://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-all-spark-notebook)
15+
- [Image Specifics :: Apache Spark](http://jupyter-docker-stacks.readthedocs.io/en/latest/using/specifics.html#apache-spark)

‎all-spark-notebook/hooks/index.tmpl

Lines changed: 0 additions & 2 deletions
This file was deleted.

0 commit comments

Comments
 (0)