Skip to content

Commit f4e80fd

Browse files
jcfrjcfr
committed
Add update and download scripts, update README
1 parent 603287b commit f4e80fd

File tree

5 files changed

+330
-0
lines changed

5 files changed

+330
-0
lines changed

‎.gitignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
MD5/
2+
SHA224/
3+
SHA256/
4+
SHA384/
5+
SHA512/

‎INCOMING/.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Ignore everything in this directory
2+
*
3+
# Except this file
4+
!.gitignore

‎README.md‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,98 @@ SlicerTestingData
22
=================
33

44
This is a mirror for the Slicer testing data.
5+
6+
To support content-based addressing, files are uploaded as assets organized in [releases](https://github.com/Slicer/SlicerTestingData/releases)
7+
named after the hashing algorithm.
8+
9+
For each hashing algorithm `<HASHALGO>`, the following are available:
10+
* release named `<HASHALGO>`
11+
* markdown document `<HASHALGO>.md` with links of the form `* [<filename>](https://github.com/Slicer/SlicerTestingData/releases/download/<HASHALGO>/<checksum>)`
12+
* a CSV file `<HASHALGO>.csv` listing all `<hashsum>;<filename>` pairs
13+
14+
Upload files
15+
------------
16+
17+
_The commands reported below should be evaluated in the same terminal session. See [Documentation conventions](#documentation-conventions)_
18+
19+
1. Install [Prerequisites](#prerequisites)
20+
21+
2. Copy files to upload in `INCOMING` directory.
22+
23+
```
24+
$ cp /path/to/files/* /path/to/SlicerTestingData/INCOMING
25+
```
26+
27+
3. If needed, create `<HASHALGO>` release
28+
29+
```
30+
$ githubrelease release Slicer/SlicerTestingData create --publish SHA256
31+
```
32+
33+
4. Run update script specifying `<HASHALGO>`, and repeat action for other `<HASHALGO>`.
34+
35+
```
36+
$ /path/to/SlicerTestingData/scripts/update.sh SHA256
37+
```
38+
39+
5. Clear content of `INCOMING` directory if all files have been uploaded for each `<HASHALGO>`.
40+
41+
6. Commit updated `<HASHALGO>.csv` and `<HASHALGO>.md` files
42+
43+
```
44+
$ hashalgo=SHA256
45+
$ git add ${hashalgo}.md ${hashalgo}.csv
46+
$ git commit -m "Update files associated with ${hashalgo}"
47+
$ git push origin master
48+
```
49+
50+
51+
Downloading files
52+
-----------------
53+
54+
_The commands reported below should be evaluated in the same terminal session. See [Documentation conventions](#documentation-conventions)_
55+
56+
1. Install [Prerequisites](#prerequisites)
57+
58+
2. Execute download script specifying `<HASHALGO>` to copy the data form
59+
60+
```
61+
$ /path/to/SlicerTestingData/scripts/update.sh SHA256
62+
```
63+
64+
65+
Prerequisites
66+
-------------
67+
68+
_The commands reported below should be evaluated in the same terminal session. See [Documentation conventions](#documentation-conventions)_
69+
70+
1. Download this project
71+
72+
```
73+
$ git clone git://github.com/Slicer/SlicerTestingData
74+
```
75+
76+
2. Install [githubrelease](https://github.com/j0057/github-release#installing)
77+
78+
```
79+
$ pip install githubrelease
80+
```
81+
82+
3. Set `GITHUB_TOKEN` env. variable. Read [here](https://github.com/j0057/github-release#configuring) for more details.
83+
84+
```
85+
$ export GITHUB_TOKEN=YOUR_TOKEN
86+
```
87+
88+
89+
Documentation conventions
90+
-------------------------
91+
92+
Commands to evaluate starts with a dollar sign. For example:
93+
94+
```
95+
$ echo "Hello"
96+
Hello
97+
```
98+
99+
means that `echo "Hello"` should be copied and evaluated in the terminal.

‎scripts/download.sh‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
#/usr/bin/env bash
2+
3+
set -e
4+
set -o pipefail
5+
6+
declare -A hashcmds=()
7+
hashcmds["MD5"]="md5sum"
8+
hashcmds["SHA224"]="sha224sum"
9+
hashcmds["SHA256"]="sha256sum"
10+
hashcmds["SHA384"]="sha384sum"
11+
hashcmds["SHA512"]="sha512sum"
12+
13+
PROG=$(basename $0)
14+
15+
script_dir=$(cd $(dirname $0) || exit 1; pwd)
16+
root_dir=$(realpath ${script_dir}/..)
17+
18+
#------------------------------------------------------------------------------
19+
if ! command -v github-release &> /dev/null; then
20+
echo >&2 'error: "github-release" not found!'
21+
exit 1
22+
fi
23+
24+
#------------------------------------------------------------------------------
25+
err() { echo -e >&2 ERROR: $@\\n; }
26+
die() { err $@; exit 1; }
27+
help() {
28+
cat >&2 <<ENDHELP
29+
Usage: $PROG HASHALGO
30+
Download files associated with HASHALGO release into directory ${root_dir}/HASHALGO/
31+
32+
ENDHELP
33+
}
34+
35+
hashalgo=$1
36+
if [[ "${hashalgo}" == "" ]]; then
37+
err Missing HASHALGO option
38+
help
39+
exit 1
40+
fi
41+
42+
hashcmd=${hashcmds[${hashalgo}]}
43+
if ! command -v ${hashcmd} &> /dev/null; then
44+
echo >&2 'error: "${hashcmd}" not found!'
45+
exit 1
46+
fi
47+
48+
if [[ ! -d ${root_dir}/INCOMING ]]; then
49+
err Missing ${root_dir}/INCOMING directory
50+
exit 1
51+
fi
52+
53+
if [[ ! -d ${root_dir}/${hashalgo} ]]; then
54+
err Missing ${root_dir}/${hashalgo} directory
55+
exit 1
56+
fi
57+
58+
if [[ ! -f ${root_dir}/${hashalgo}.csv ]]; then
59+
err Missing ${root_dir}/${hashalgo}.csv file
60+
exit 1
61+
fi
62+
63+
#------------------------------------------------------------------------------
64+
echo "${hashalgo}: downloading release assets"
65+
pushd ${root_dir}/${hashalgo} > /dev/null
66+
for line in $(cat ${root_dir}/${hashalgo}.csv); do
67+
checksum=$(echo ${line} | cut -d";" -f1)
68+
filename=$(echo ${line} | cut -d";" -f2)
69+
download=1
70+
if [[ -f ${checksum} ]]; then
71+
current_checksum=$(${hashcmd} ${checksum} | awk '{ print $1 }')
72+
if [[ ${checksum} == ${current_checksum} ]]; then
73+
download=0
74+
fi
75+
fi
76+
if [[ $download -eq 1 ]]; then
77+
githubrelease asset Slicer/SlicerTestingData download SHA256 ${checksum}
78+
echo
79+
else
80+
echo "${hashalgo}: skipping download ${checksum}"
81+
fi
82+
done
83+
popd > /dev/null
84+
echo
85+
86+
#------------------------------------------------------------------------------
87+
echo "${hashalgo}: copying back to INCOMING directory"
88+
for line in $(cat ${root_dir}/${hashalgo}.csv); do
89+
checksum=$(echo ${line} | cut -d";" -f1)
90+
filename=$(echo ${line} | cut -d";" -f2)
91+
echo "${hashalgo}: copying ${filename}"
92+
cp ${root_dir}/${hashalgo}/${checksum} ${root_dir}/INCOMING/${filename}
93+
done
94+

‎scripts/update.sh‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
#/usr/bin/env bash
2+
3+
set -e
4+
set -o pipefail
5+
6+
PROG=$(basename $0)
7+
8+
script_dir=$(cd $(dirname $0) || exit 1; pwd)
9+
root_dir=$(realpath ${script_dir}/..)
10+
11+
declare -A hashcmds=()
12+
hashcmds["MD5"]="md5sum"
13+
hashcmds["SHA224"]="sha224sum"
14+
hashcmds["SHA256"]="sha256sum"
15+
hashcmds["SHA384"]="sha384sum"
16+
hashcmds["SHA512"]="sha512sum"
17+
18+
supported_hashalgos=${!hashcmds[@]}
19+
20+
#------------------------------------------------------------------------------
21+
if ! command -v github-release &> /dev/null; then
22+
echo >&2 'error: "github-release" not found!'
23+
exit 1
24+
fi
25+
26+
#------------------------------------------------------------------------------
27+
err() { echo -e >&2 ERROR: $@\\n; }
28+
die() { err $@; exit 1; }
29+
help() {
30+
cat >&2 <<ENDHELP
31+
Usage: $PROG HASHALGO [HASHALGO ...]
32+
33+
Upload incoming files associated them with HASHALGO release.
34+
Available HASHALGO are ${supported_hashalgos}
35+
36+
ENDHELP
37+
}
38+
39+
if [[ "$@" == "" ]]; then
40+
err Missing HASHALGO option
41+
help
42+
exit 1
43+
fi
44+
for hashalgo in "$@"; do
45+
if [[ ! "${supported_hashalgos}" =~ "${hashalgo}" ]]; then
46+
err Unknown HASHALGO [${hashalgo}] option
47+
help
48+
exit 1
49+
fi
50+
done
51+
hashalgos="$@"
52+
53+
for hashalgo in ${hashalgos}; do
54+
hashcmd=${hashcmds[${hashalgo}]}
55+
if ! command -v ${hashcmd} &> /dev/null; then
56+
echo >&2 'error: "${hashcmd}" not found!'
57+
exit 1
58+
fi
59+
done
60+
61+
#------------------------------------------------------------------------------
62+
for hashalgo in ${hashalgos}; do
63+
64+
hashcmd=${hashcmds[${hashalgo}]}
65+
66+
mkdir -p ${root_dir}/${hashalgo}
67+
68+
if [[ ! -f ${root_dir}/${hashalgo}.csv ]]; then
69+
touch ${root_dir}/${hashalgo}.csv
70+
fi
71+
72+
#----
73+
echo "${hashalgo}: checking for incoming files"
74+
pushd ${root_dir}/INCOMING > /dev/null
75+
for filename in $(ls -1 | sort); do
76+
echo "${hashalgo}: found ${filename}"
77+
checksum=$(${hashcmd} ${filename} | awk '{ print $1 }')
78+
cp ${filename} ${root_dir}/${hashalgo}/${checksum}
79+
referenced_assets=$(cat ${root_dir}/${hashalgo}.csv | cut -d";" -f1)
80+
if [[ ! "${referenced_assets}" =~ "${checksum}" ]]; then
81+
echo "${checksum};${filename}" >> ${root_dir}/${hashalgo}.csv
82+
fi
83+
done
84+
popd > /dev/null
85+
echo
86+
87+
#----
88+
echo "${hashalgo}: retrieving list of uploaded assets"
89+
set +o pipefail
90+
uploaded_assets=$(
91+
githubrelease asset Slicer/SlicerTestingData list ${hashalgo} | tr -d " " | grep name -A1 | sed -e "s/name://" | sed -e "s/state://" | grep -v -E "\-\-" | while read checksum
92+
do
93+
read upload_state
94+
if [[ ${upload_state} == "uploaded" ]]; then
95+
echo ${checksum}
96+
else
97+
# Remove asset partially uploaded
98+
githubrelease asset Slicer/SlicerTestingData delete ${hashalgo} ${checksum}
99+
fi
100+
done
101+
)
102+
set -o pipefail
103+
echo
104+
105+
rm -f ${root_dir}/${hashalgo}.md
106+
107+
#----
108+
echo "${hashalgo}: uploading release assets"
109+
for line in $(cat ${root_dir}/${hashalgo}.csv); do
110+
checksum=$(echo ${line} | cut -d";" -f1)
111+
filename=$(echo ${line} | cut -d";" -f2)
112+
echo "checksum [${checksum}]"
113+
if [[ "${uploaded_assets}" =~ "${checksum}" ]]; then
114+
echo "${hashalgo}: skipping ${checksum}"
115+
else
116+
githubrelease asset Slicer/SlicerTestingData upload ${hashalgo} ${root_dir}/${hashalgo}/${checksum}
117+
fi
118+
echo "* [$filename](https://github.com/Slicer/SlicerTestingData/releases/download/${hashalgo}/${checksum})" >> ${root_dir}/${hashalgo}.md
119+
done
120+
echo
121+
122+
#----
123+
echo "${hashalgo}: updating ${hashalgo}.csv and ${hashalgo}.md"
124+
githubrelease asset Slicer/SlicerTestingData delete ${hashalgo} "${hashalgo}.csv ${hashalgo}.md"
125+
githubrelease asset Slicer/SlicerTestingData upload ${hashalgo} ${root_dir}/${hashalgo}.csv ${root_dir}/${hashalgo}.md
126+
echo
127+
128+
#----
129+
echo "${hashalgo}: updating release notes"
130+
githubrelease release Slicer/SlicerTestingData edit --body "$(cat ${root_dir}/${hashalgo}.md)" ${hashalgo}
131+
132+
done

0 commit comments

Comments
 (0)