[FLINK-30237][coordination] Only bundle single ZK client

Author: zentol

docs/content.zh/docs/deployment/ha/zookeeper_ha.md

@@ -111,16 +111,6 @@ Enabling this feature will make Flink more resilient against temporary connectio
 
 For more information take a look at [Curator's error handling](https://curator.apache.org/errors.html).
 
-## ZooKeeper 版本
-
-Flink 附带了 3.4 和 3.5 的单独的 ZooKeeper 客户端，其中 3.4 位于发行版的 `lib` 目录中，为默认使用版本，而 3.5 位于 opt 目录中。
-
-3.5 客户端允许你通过 SSL 保护 ZooKeeper 连接，但 _可能_ 不适用于 3.4 版本的 ZooKeeper 安装。
-
-你可以通过在 `lib` 目录中放置任意一个 jar 来控制 Flink 使用哪个版本。
-
-{{< top >}}
-
 <a name="bootstrap-zookeeper" />
 
 ## 启动 ZooKeeper

docs/content/docs/deployment/ha/zookeeper_ha.md

@@ -118,17 +118,6 @@ Enabling this feature will make Flink more resilient against temporary connectio
 
 For more information take a look at [Curator's error handling](https://curator.apache.org/errors.html).
 
-## ZooKeeper Versions
-
-Flink ships with separate ZooKeeper clients for 3.4 and 3.5, with 3.4 being in the `lib` directory of the distribution
-and thus used by default, whereas 3.5 is placed in the `opt` directory.
-
-The 3.5 client allows you to secure the ZooKeeper connection via SSL, but _may_ not work with 3.4- ZooKeeper installations.
-
-You can control which version is used by Flink by placing either jar in the `lib` directory.
-
-{{< top >}}
-
 ## Bootstrap ZooKeeper
 
 If you don't have a running ZooKeeper installation, you can use the helper scripts, which ship with Flink.

flink-dist/pom.xml

@@ -33,7 +33,6 @@ under the License.
 	<packaging>jar</packaging>
 
 	<properties>
-		<zookeeper.optional.version>3.6.3</zookeeper.optional.version>
 		<japicmp.skip>true</japicmp.skip>
 	</properties>
 
@@ -628,25 +627,6 @@ under the License.
 							<outputDirectory>${project.build.directory}/temporary/java11_exclusive</outputDirectory>
 						</configuration>
 					</execution>
-					<execution>
-						<id>copy-zk-jars</id>
-						<phase>process-resources</phase>
-						<goals>
-							<goal>copy</goal>
-						</goals>
-						<configuration>
-							<artifactItems>
-								<artifactItem>
-									<groupId>org.apache.flink</groupId>
-									<artifactId>flink-shaded-zookeeper-3</artifactId>
-									<version>${zookeeper.optional.version}-${flink.shaded.version}</version>
-									<type>jar</type>
-									<overWrite>true</overWrite>
-								</artifactItem>
-							</artifactItems>
-							<outputDirectory>${project.build.directory}/temporary</outputDirectory>
-						</configuration>
-					</execution>
 				</executions>
 			</plugin>
 
@@ -702,8 +682,6 @@ under the License.
 								<excludes>
 									<!-- log4j 2 is bundled separately from the flink-dist jar -->
 									<exclude>org.apache.logging.log4j:*</exclude>
-									<!-- Bundled separately so that users can easily switch between ZK 3.4/3.5-->
-									<exclude>org.apache.flink:flink-shaded-zookeeper-3</exclude>
 								</excludes>
 							</artifactSet>
 						</configuration>

flink-dist/src/main/assemblies/bin.xml

@@ -44,14 +44,6 @@ under the License.
 				<include>org.apache.logging.log4j:log4j-1.2-api</include>
 			</includes>
 		</dependencySet>
-		<dependencySet>
-			<outputDirectory>lib</outputDirectory>
-			<unpack>false</unpack>
-			<includes>
-				<include>org.apache.flink:flink-shaded-zookeeper-3:jar:${zookeeper.version}-${flink.shaded.version}</include>
-			</includes>
-			<outputFileNameMapping>flink-shaded-zookeeper-${zookeeper.version}.jar</outputFileNameMapping>
-		</dependencySet>
 	</dependencySets>
 
 	<files>

flink-dist/src/main/assemblies/opt.xml

@@ -115,13 +115,6 @@
 			<destName>flink-python-${project.version}.jar</destName>
 			<fileMode>0644</fileMode>
 		</file>
-
-		<file>
-			<source>target/temporary/flink-shaded-zookeeper-3-${zookeeper.optional.version}-${flink.shaded.version}.jar</source>
-			<outputDirectory>opt</outputDirectory>
-			<destName>flink-shaded-zookeeper-${zookeeper.optional.version}.jar</destName>
-			<fileMode>0644</fileMode>
-		</file>
 	</files>
 	<fileSets>
 		<fileSet>

flink-end-to-end-tests/run-nightly-tests.sh

@@ -162,15 +162,15 @@ function run_group_1 {
 
     run_test "Running HA dataset end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_dataset.sh" "skip_check_exceptions"
 
-    run_test "Running HA (hashmap, async) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh hashmap true false 3.6" "skip_check_exceptions"
-    run_test "Running HA (hashmap, sync) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh hashmap false false 3.5" "skip_check_exceptions"
-    run_test "Running HA (rocks, non-incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh rocks true false 3.6" "skip_check_exceptions"
-    run_test "Running HA (rocks, incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh rocks true true 3.5" "skip_check_exceptions"
-
-    run_test "Running HA per-job cluster (hashmap, async) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh hashmap true false 3.6" "skip_check_exceptions"
-    run_test "Running HA per-job cluster (hashmap, sync) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh hashmap false false 3.5" "skip_check_exceptions"
-    run_test "Running HA per-job cluster (rocks, non-incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh rocks true false 3.6" "skip_check_exceptions"
-    run_test "Running HA per-job cluster (rocks, incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh rocks true true 3.5" "skip_check_exceptions"
+    run_test "Running HA (hashmap, async) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh hashmap true false" "skip_check_exceptions"
+    run_test "Running HA (hashmap, sync) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh hashmap false false" "skip_check_exceptions"
+    run_test "Running HA (rocks, non-incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh rocks true false" "skip_check_exceptions"
+    run_test "Running HA (rocks, incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh rocks true true" "skip_check_exceptions"
+
+    run_test "Running HA per-job cluster (hashmap, async) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh hashmap true false" "skip_check_exceptions"
+    run_test "Running HA per-job cluster (hashmap, sync) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh hashmap false false" "skip_check_exceptions"
+    run_test "Running HA per-job cluster (rocks, non-incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh rocks true false" "skip_check_exceptions"
+    run_test "Running HA per-job cluster (rocks, incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh rocks true true" "skip_check_exceptions"
 }
 
 function run_group_2 {

flink-end-to-end-tests/test-scripts/common.sh

@@ -114,22 +114,6 @@ function revert_flink_dir() {
     CURL_SSL_ARGS=""
 }
 
-function setup_flink_shaded_zookeeper() {
-  local version=$1
-  # if it is already in lib we don't have to do anything
-  if ! [ -e "${FLINK_DIR}"/lib/flink-shaded-zookeeper-${version}* ]; then
-    if ! [ -e "${FLINK_DIR}"/opt/flink-shaded-zookeeper-${version}* ]; then
-      echo "Could not find ZK ${version} in opt or lib."
-      exit 1
-    else
-      # contents of 'opt' must not be changed since it is not backed up in common.sh#backup_flink_dir
-      # it is fine to delete jars from 'lib' since it is backed up and will be restored after the test
-      rm "${FLINK_DIR}"/lib/flink-shaded-zookeeper-*
-      cp "${FLINK_DIR}"/opt/flink-shaded-zookeeper-${version}* "${FLINK_DIR}/lib"
-    fi
-  fi
-}
-
 function add_optional_lib() {
     local lib_name=$1
     cp "$FLINK_DIR/opt/flink-${lib_name}"*".jar" "$FLINK_DIR/lib"

flink-end-to-end-tests/test-scripts/test_ha_datastream.sh

@@ -34,7 +34,6 @@ function run_ha_test() {
     local BACKEND=$2
     local ASYNC=$3
     local INCREM=$4
-    local ZOOKEEPER_VERSION=$5
 
     local JM_KILLS=2
     local CHECKPOINT_DIR="${TEST_DATA_DIR}/checkpoints/"
@@ -47,11 +46,10 @@ function run_ha_test() {
     # jm killing loop
     set_config_key "env.pid.dir" "${TEST_DATA_DIR}"
     set_config_key "env.java.opts.all" "-ea"
-    setup_flink_shaded_zookeeper ${ZOOKEEPER_VERSION}
     start_local_zk
     start_cluster
 
-    echo "Running on HA mode: parallelism=${PARALLELISM}, backend=${BACKEND}, asyncSnapshots=${ASYNC}, incremSnapshots=${INCREM} and zk=${ZOOKEEPER_VERSION}."
+    echo "Running on HA mode: parallelism=${PARALLELISM}, backend=${BACKEND}, asyncSnapshots=${ASYNC}, incremSnapshots=${INCREM}."
 
     # submit a job in detached mode and let it run
     local JOB_ID=$($FLINK_DIR/bin/flink run -d -p ${PARALLELISM} \
@@ -98,6 +96,5 @@ function run_ha_test() {
 STATE_BACKEND_TYPE=${1:-file}
 STATE_BACKEND_FILE_ASYNC=${2:-true}
 STATE_BACKEND_ROCKS_INCREMENTAL=${3:-false}
-ZOOKEEPER_VERSION=${4:-3.5}
 
-run_test_with_timeout 900 run_ha_test 4 ${STATE_BACKEND_TYPE} ${STATE_BACKEND_FILE_ASYNC} ${STATE_BACKEND_ROCKS_INCREMENTAL} ${ZOOKEEPER_VERSION}
+run_test_with_timeout 900 run_ha_test 4 ${STATE_BACKEND_TYPE} ${STATE_BACKEND_FILE_ASYNC} ${STATE_BACKEND_ROCKS_INCREMENTAL}

flink-end-to-end-tests/test-scripts/test_ha_per_job_cluster_datastream.sh

@@ -97,7 +97,6 @@ function run_ha_test() {
     local BACKEND=$2
     local ASYNC=$3
     local INCREM=$4
-    local ZOOKEEPER_VERSION=$5
 
     local JM_KILLS=2
 
@@ -113,10 +112,9 @@ function run_ha_test() {
     # jm killing loop
     set_config_key "env.pid.dir" "${TEST_DATA_DIR}"
 
-    setup_flink_shaded_zookeeper ${ZOOKEEPER_VERSION}
     start_local_zk
 
-    echo "Running on HA mode: parallelism=${PARALLELISM}, backend=${BACKEND}, asyncSnapshots=${ASYNC}, incremSnapshots=${INCREM} and zk=${ZOOKEEPER_VERSION}."
+    echo "Running on HA mode: parallelism=${PARALLELISM}, backend=${BACKEND}, asyncSnapshots=${ASYNC}, incremSnapshots=${INCREM}."
 
     # submit a job in detached mode and let it run
     run_job ${PARALLELISM} ${BACKEND} ${ASYNC} ${INCREM}
@@ -154,6 +152,5 @@ function run_ha_test() {
 STATE_BACKEND_TYPE=${1:-file}
 STATE_BACKEND_FILE_ASYNC=${2:-true}
 STATE_BACKEND_ROCKS_INCREMENTAL=${3:-false}
-ZOOKEEPER_VERSION=${4:-3.5}
 
-run_test_with_timeout 900 run_ha_test 4 ${STATE_BACKEND_TYPE} ${STATE_BACKEND_FILE_ASYNC} ${STATE_BACKEND_ROCKS_INCREMENTAL} ${ZOOKEEPER_VERSION}
+run_test_with_timeout 900 run_ha_test 4 ${STATE_BACKEND_TYPE} ${STATE_BACKEND_FILE_ASYNC} ${STATE_BACKEND_ROCKS_INCREMENTAL}