Flink 16411 clean for pr

.travis.yml

@@ -59,7 +59,8 @@ before_script:
 
 # Install maven 3.2.5 since xenial uses 3.3.9 for which shading is broken
 before_install:
-   - source ./tools/travis/setup_maven.sh
+   - source ./tools/ci/maven-utils.sh
+   - setup_maven
 
 notifications:
   slack:

flink-end-to-end-tests/run-nightly-tests.sh

@@ -32,6 +32,7 @@ if [ -z "$FLINK_DIR" ] ; then
     exit 1
 fi
 
+source "${END_TO_END_DIR}/../tools/ci/maven-utils.sh"
 source "${END_TO_END_DIR}/test-scripts/test-runner-common.sh"
 
 # On Azure CI, set artifacts dir
@@ -57,7 +58,7 @@ echo "Flink distribution directory: $FLINK_DIR"
 
 echo "Java and Maven version"
 java -version
-mvn -version
+run_mvn -version
 
 echo "Free disk space"
 df -h
@@ -230,13 +231,13 @@ printf "========================================================================
 
 LOG4J_PROPERTIES=${END_TO_END_DIR}/../tools/log4j-travis.properties
 
-MVN_LOGGING_OPTIONS="-Dlog.dir=${ARTIFACTS_DIR} -DlogBackupDir=${ARTIFACTS_DIR} -Dlog4j.configurationFile=file://$LOG4J_PROPERTIES -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
-MVN_COMMON_OPTIONS="-nsu -B -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.wagon.http.pool=false -Dfast -Pskip-webui-build"
+MVN_LOGGING_OPTIONS="-Dlog.dir=${ARTIFACTS_DIR} -DlogBackupDir=${ARTIFACTS_DIR} -Dlog4j.configurationFile=file://$LOG4J_PROPERTIES"
+MVN_COMMON_OPTIONS="-Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dfast -Pskip-webui-build"
 e2e_modules=$(find flink-end-to-end-tests -mindepth 2 -maxdepth 5 -name 'pom.xml' -printf '%h\n' | sort -u | tr '\n' ',')
 e2e_modules="${e2e_modules},$(find flink-walkthroughs -mindepth 2 -maxdepth 2 -name 'pom.xml' -printf '%h\n' | sort -u | tr '\n' ',')"
 
 PROFILE="$PROFILE -Pe2e-travis1 -Pe2e-travis2 -Pe2e-travis3 -Pe2e-travis4 -Pe2e-travis5 -Pe2e-travis6"
-mvn ${MVN_COMMON_OPTIONS} ${MVN_LOGGING_OPTIONS} ${PROFILE} verify -pl ${e2e_modules} -DdistDir=$(readlink -e build-target)
+run_mvn ${MVN_COMMON_OPTIONS} ${MVN_LOGGING_OPTIONS} ${PROFILE} verify -pl ${e2e_modules} -DdistDir=$(readlink -e build-target)
 
 EXIT_CODE=$?
 

flink-end-to-end-tests/run-single-test.sh

@@ -47,6 +47,7 @@ if [ -z "$FLINK_DIR" ] ; then
 fi
 
 source "${END_TO_END_DIR}/test-scripts/test-runner-common.sh"
+source "${END_TO_END_DIR}/../tools/ci/maven-utils.sh"
 
 cleanup_tmp_files
 

flink-end-to-end-tests/test-scripts/common_ssl.sh

@@ -80,7 +80,7 @@ function _set_conf_ssl_helper {
         git clone https://github.com/apache/flink-shaded.git
         cd flink-shaded
         git checkout "release-${FLINK_SHADED_VERSION}"
-        mvn clean package -Pinclude-netty-tcnative-static -pl flink-shaded-netty-tcnative-static
+        run_mvn clean package -Pinclude-netty-tcnative-static -pl flink-shaded-netty-tcnative-static
         cp flink-shaded-netty-tcnative-static/target/flink-shaded-netty-tcnative-static-*.jar $FLINK_DIR/lib/
         cd ..
         rm -rf flink-shaded

flink-end-to-end-tests/test-scripts/test-runner-common.sh

@@ -19,7 +19,7 @@
 
 source "${END_TO_END_DIR}"/test-scripts/common.sh
 
-export FLINK_VERSION=$(mvn --file ${END_TO_END_DIR}/pom.xml org.apache.maven.plugins:maven-help-plugin:3.1.0:evaluate -Dexpression=project.version -q -DforceStdout)
+export FLINK_VERSION=$(MVN_RUN_VERBOSE=false run_mvn --file ${END_TO_END_DIR}/pom.xml org.apache.maven.plugins:maven-help-plugin:3.1.0:evaluate -Dexpression=project.version -q -DforceStdout)
 
 #######################################
 # Prints the given description, runs the given test and prints how long the execution took.

flink-end-to-end-tests/test-scripts/test_datastream_walkthroughs.sh

@@ -31,7 +31,7 @@ cd "${TEST_DATA_DIR}"
 ARTIFACT_ID=flink-walkthrough-datastream-${TEST_TYPE}
 ARTIFACT_VERSION=0.1
 
-mvn archetype:generate                                          \
+run_mvn archetype:generate                                          \
     -DarchetypeGroupId=org.apache.flink                         \
     -DarchetypeArtifactId=flink-walkthrough-datastream-${TEST_TYPE}  \
     -DarchetypeVersion=${FLINK_VERSION}                         \
@@ -44,7 +44,7 @@ mvn archetype:generate                                          \
 
 cd "${ARTIFACT_ID}"
 
-mvn clean package -nsu > compile-output.txt
+run_mvn clean package > compile-output.txt
 
 if [[ `grep -c "BUILD FAILURE" compile-output.txt` -eq '1' ]]; then
     echo "Failure: The walkthrough did not successfully compile"

flink-end-to-end-tests/test-scripts/test_quickstarts.sh

@@ -43,7 +43,7 @@ cd "${TEST_DATA_DIR}"
 ARTIFACT_ID=flink-quickstart-${TEST_TYPE}
 ARTIFACT_VERSION=0.1
 
-mvn archetype:generate                                   \
+run_mvn archetype:generate                                   \
     -DarchetypeGroupId=org.apache.flink                  \
     -DarchetypeArtifactId=flink-quickstart-${TEST_TYPE}  \
     -DarchetypeVersion=${FLINK_VERSION}                  \
@@ -68,7 +68,7 @@ sed -i -e ''$(($position + 1))'i\
 
 sed -i -e "s/org.apache.flink.quickstart.StreamingJob/org.apache.flink.quickstart.$TEST_CLASS_NAME/" pom.xml
 
-mvn clean package -nsu
+run_mvn clean package
 
 cd target
 jar tvf flink-quickstart-${TEST_TYPE}-0.1.jar > contentsInJar.txt

flink-end-to-end-tests/test-scripts/test_streaming_gcp_pubsub.sh

@@ -19,4 +19,4 @@
 
 cd "${END_TO_END_DIR}/flink-connector-gcp-pubsub-emulator-tests"
 
-mvn test -DskipTests=false
+run_mvn test -DskipTests=false

flink-end-to-end-tests/test-scripts/test_table_walkthroughs.sh

@@ -31,7 +31,7 @@ cd "${TEST_DATA_DIR}"
 ARTIFACT_ID=flink-walkthrough-table-${TEST_TYPE}
 ARTIFACT_VERSION=0.1
 
-mvn archetype:generate                                          \
+run_mvn archetype:generate                                          \
     -DarchetypeGroupId=org.apache.flink                         \
     -DarchetypeArtifactId=flink-walkthrough-table-${TEST_TYPE}  \
     -DarchetypeVersion=${FLINK_VERSION}                         \
@@ -44,7 +44,7 @@ mvn archetype:generate                                          \
 
 cd "${ARTIFACT_ID}"
 
-mvn clean package -nsu > compile-output.txt
+run_mvn clean package > compile-output.txt
 
 if [[ `grep -c "BUILD FAILURE" compile-output.txt` -eq '1' ]]; then
     echo "Failure: The walk-through did not successfully compile"

tools/azure-pipelines/jobs-template.yml

@@ -160,14 +160,16 @@ jobs:
         echo "##vso[task.setvariable variable=PATH]$JAVA_HOME_11_X64/bin:$PATH"
       displayName: "Set to jdk11"
       condition: eq('${{parameters.jdk}}', 'jdk11')
-    - script: ./tools/travis/setup_maven.sh
+    - script: |
+        source ./tools/ci/maven-utils.sh
+        setup_maven
       displayName: Setup Maven 3.2.5
     - script: ./tools/azure-pipelines/setup_docker.sh
       displayName: Setup Docker
     - script: ./tools/azure-pipelines/free_disk_space.sh
       displayName: Free up disk space
     - script: sudo apt-get install -y bc
-    - script: M2_HOME=/home/vsts/maven_cache/apache-maven-3.2.5/ PATH=/home/vsts/maven_cache/apache-maven-3.2.5/bin:$PATH ${{parameters.environment}} STAGE=compile ./tools/azure_controller.sh compile
+    - script: ${{parameters.environment}} STAGE=compile ./tools/azure_controller.sh compile
       displayName: Build Flink
     # TODO remove pre-commit tests script by adding the tests to the nightly script
 #    - script: FLINK_DIR=build-target ./flink-end-to-end-tests/run-pre-commit-tests.sh

tools/azure-pipelines/prepare_precommit.sh

@@ -31,9 +31,8 @@ find . -type f -name '*.timestamp' | xargs touch
 
 export M2_HOME=/home/vsts/maven_cache/apache-maven-3.2.5/ 
 export PATH=/home/vsts/maven_cache/apache-maven-3.2.5/bin:$PATH
-mvn -version
-MVN_CALL="mvn install --settings ./tools/azure-pipelines/google-mirror-settings.xml -DskipTests -Drat.skip $PROFILE"
-echo "Invoking Maven: '$MVN_CALL'"
+run_mvn -version
+MVN_CALL="run_mvn install -DskipTests -Drat.skip"
 $MVN_CALL
 EXIT_CODE=$?
 

tools/azure_controller.sh

@@ -17,15 +17,6 @@
 # limitations under the License.
 ################################################################################
 
-echo $M2_HOME
-echo $PATH
-echo $MAVEN_OPTS
-
-mvn -version
-echo "Commit: $(git rev-parse HEAD)"
-
-
-
 HERE="`dirname \"$0\"`"             # relative
 HERE="`( cd \"$HERE\" && pwd )`"    # absolutized and normalized
 if [ -z "$HERE" ] ; then
@@ -36,6 +27,14 @@ fi
 
 source "${HERE}/travis/stage.sh"
 source "${HERE}/travis/shade.sh"
+source "${HERE}/ci/maven-utils.sh"
+
+echo $M2_HOME
+echo $PATH
+echo $MAVEN_OPTS
+
+run_mvn -version
+echo "Commit: $(git rev-parse HEAD)"
 
 print_system_info() {
     echo "CPU information"
@@ -63,8 +62,7 @@ EXIT_CODE=0
 # mirror. We use a different mirror because the official maven central mirrors
 # often lead to connection timeouts (probably due to rate-limiting)
 
-# adding -Dmaven.wagon.http.pool=false (see https://developercommunity.visualstudio.com/content/problem/851041/microsoft-hosted-agents-run-into-maven-central-tim.html)
-MVN="mvn clean install --settings ./tools/azure-pipelines/google-mirror-settings.xml $MAVEN_OPTS -nsu -Dflink.convergence.phase=install -Pcheck-convergence -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.wagon.http.pool=false -Dmaven.javadoc.skip=true -B -U -DskipTests $PROFILE"
+MVN="run_mvn clean install $MAVEN_OPTS -Dflink.convergence.phase=install -Pcheck-convergence -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.javadoc.skip=true -U -DskipTests"
 
 # Run actual compile&test steps
 if [ $STAGE == "$STAGE_COMPILE" ]; then

tools/ci/maven-utils.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CI_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+MAVEN_VERSION="3.2.5"
+MAVEN_CACHE_DIR=${HOME}/maven_cache
+MAVEN_VERSIONED_DIR=${MAVEN_CACHE_DIR}/apache-maven-${MAVEN_VERSION}
+
+export MVN_GLOBAL_OPTIONS=""
+# see https://developercommunity.visualstudio.com/content/problem/851041/microsoft-hosted-agents-run-into-maven-central-tim.html
+MVN_GLOBAL_OPTIONS+="-Dmaven.wagon.http.pool=false "
+# use google mirror everywhere
+MVN_GLOBAL_OPTIONS+="--settings $CI_DIR/google-mirror-settings.xml "
+# logging 
+MVN_GLOBAL_OPTIONS+="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn "
+# suppress snapshot updates
+MVN_GLOBAL_OPTIONS+="--no-snapshot-updates "
+# enable non-interactive batch mode
+MVN_GLOBAL_OPTIONS+="-B "
+# globally control the build profile details
+MVN_GLOBAL_OPTIONS+="$PROFILE "
+
+# Utility for invoking Maven in CI
+function run_mvn {
+	MVN_CMD="mvn"
+	if [[ "$M2_HOME" != "" ]]; then
+		MVN_CMD="${M2_HOME}/bin/mvn"
+	fi
+
+	ARGS=$@
+	INVOCATION="$MVN_CMD $MVN_GLOBAL_OPTIONS $ARGS"
+	if [[ "$MVN_RUN_VERBOSE" != "false" ]]; then
+		echo "Invoking mvn with '$INVOCATION'"
+	fi
+	${INVOCATION}
+}
+export -f run_mvn
+
+function setup_maven {
+	if [ ! -d "${MAVEN_VERSIONED_DIR}" ]; then
+	  wget https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.zip
+	  unzip -d "${MAVEN_CACHE_DIR}" -qq "apache-maven-${MAVEN_VERSION}-bin.zip"
+	  rm "apache-maven-${MAVEN_VERSION}-bin.zip"
+	fi
+
+	export M2_HOME="${MAVEN_VERSIONED_DIR}"
+	echo "##vso[task.setvariable variable=M2_HOME]$M2_HOME"
+
+	# just in case: clean up the .m2 home and remove invalid jar files
+	if [ -d "${HOME}/.m2/repository/" ]; then
+	  find ${HOME}/.m2/repository/ -name "*.jar" -exec sh -c 'if ! zip -T {} >/dev/null ; then echo "deleting invalid file: {}"; rm -f {} ; fi' \;
+	fi
+
+	echo "Installed Maven ${MAVEN_VERSION} to ${M2_HOME}"
+}

tools/travis_controller.sh

@@ -90,7 +90,7 @@ EXIT_CODE=0
 
 # Run actual compile&test steps
 if [ $STAGE == "$STAGE_COMPILE" ]; then
-	MVN="mvn clean install -nsu -Dflink.convergence.phase=install -Pcheck-convergence -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.javadoc.skip=true -B -DskipTests $PROFILE -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -Dmaven.wagon.http.pool=false"
+	MVN="run_mvn clean install -Dflink.convergence.phase=install -Pcheck-convergence -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.javadoc.skip=true -DskipTests"
 	$MVN
 	EXIT_CODE=$?
 

tools/travis_watchdog.sh

@@ -26,6 +26,7 @@ if [ -z "$HERE" ] ; then
 fi
 
 source "${HERE}/travis/stage.sh"
+source "${HERE}/ci/maven-utils.sh"
 
 ARTIFACTS_DIR="${HERE}/artifacts"
 
@@ -63,21 +64,17 @@ MVN_TEST_MODULES=$(get_test_modules_for_stage ${TEST})
 # Maven command to run. We set the forkCount manually, because otherwise Maven sees too many cores
 # on the Travis VMs. Set forkCountTestPackage to 1 for container-based environment (4 GiB memory)
 # and 2 for sudo-enabled environment (7.5 GiB memory).
-#
-# -nsu option forbids downloading snapshot artifacts. The only snapshot artifacts we depend are from
-# Flink, which however should all be built locally. see FLINK-7230
-#
-MVN_LOGGING_OPTIONS="-Dlog.dir=${ARTIFACTS_DIR} -Dlog4j.configurationFile=file://$LOG4J_PROPERTIES -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
-MVN_COMMON_OPTIONS="-nsu -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dfast -Dmaven.wagon.http.pool=false -B -Pskip-webui-build $MVN_LOGGING_OPTIONS"
+MVN_LOGGING_OPTIONS="-Dlog.dir=${ARTIFACTS_DIR} -Dlog4j.configurationFile=file://$LOG4J_PROPERTIES"
+MVN_COMMON_OPTIONS="-Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dfast -Pskip-webui-build $MVN_LOGGING_OPTIONS"
 MVN_COMPILE_OPTIONS="-DskipTests"
 MVN_TEST_OPTIONS="-Dflink.tests.with-openssl"
 
 e2e_modules=$(find flink-end-to-end-tests -mindepth 2 -maxdepth 5 -name 'pom.xml' -printf '%h\n' | sort -u | tr '\n' ',')
 
-MVN_COMPILE="mvn $MVN_COMMON_OPTIONS $MVN_COMPILE_OPTIONS $PROFILE $MVN_COMPILE_MODULES install"
-MVN_TEST="mvn $MVN_COMMON_OPTIONS $MVN_TEST_OPTIONS $PROFILE $MVN_TEST_MODULES verify"
+MVN_COMPILE="run_mvn $MVN_COMMON_OPTIONS $MVN_COMPILE_OPTIONS $PROFILE $MVN_COMPILE_MODULES install"
+MVN_TEST="run_mvn $MVN_COMMON_OPTIONS $MVN_TEST_OPTIONS $PROFILE $MVN_TEST_MODULES verify"
 # don't move the e2e-pre-commit profile activation into the misc entry in .travis.yml, since it breaks caching
-MVN_E2E="mvn $MVN_COMMON_OPTIONS $MVN_TEST_OPTIONS $PROFILE -Pe2e-pre-commit -pl ${e2e_modules},flink-dist verify"
+MVN_E2E="run_mvn $MVN_COMMON_OPTIONS $MVN_TEST_OPTIONS -Pe2e-pre-commit -pl ${e2e_modules},flink-dist verify"
 
 MVN_PID="${ARTIFACTS_DIR}/watchdog.mvn.pid"
 MVN_EXIT="${ARTIFACTS_DIR}/watchdog.mvn.exit"

tools/verify_scala_suffixes.sh

@@ -117,7 +117,7 @@ while read line; do
             block_infected=1
         fi
     fi
-done < <(mvn -nsu dependency:tree -Dincludes=org.scala-lang,:*_2.1*:: -pl ${excluded_modules} ${MAVEN_ARGUMENTS} | tee /dev/tty)
+done < <(run_mvn dependency:tree -Dincludes=org.scala-lang,:*_2.1*:: -pl ${excluded_modules} ${MAVEN_ARGUMENTS} | tee /dev/tty)
 
 
 # deduplicate and sort