Skip to content

Commit 8d26a5c

Browse files
andrewor14andrewor14
committed
Add tests for bash/utils.sh
1 parent 4ae24c3 commit 8d26a5c

File tree

2 files changed

+250
-0
lines changed

2 files changed

+250
-0
lines changed

‎bin/test.conf‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
# --------------------------------------------------------------------------------------------
2+
# Spark properties file for testing
3+
#
4+
# The configs are separated into three categories, one for each delimiter that Java supports:
5+
# http://docs.oracle.com/javase/7/docs/api/java/util/Properties.html#load(java.io.Reader).
6+
# The values of the configs are assumed to be identical across the categories. Changes in
7+
# this file must be synced with "bin/test.sh"
8+
# --------------------------------------------------------------------------------------------
9+
10+
# Space delimiter
11+
spark.space.1 -Dstraw=berry
12+
spark.space.2 -Dstraw="berry"
13+
spark.space.3 -Dstraw="berry again"
14+
spark.space.4 -Dstraw="berry \"quote"
15+
spark.space.5 -Dstraw="berry \\backslash"
16+
spark.space.6 -Dstraw="berry \"quotes\" and \\backslashes\\ "
17+
spark.space.7 -Dstraw=berry -Dblue=berry -Dblack=berry
18+
spark.space.8 -Dstraw="berry space" -Dblue="berry" -Dblack=berry
19+
spark.space.9 -Dstraw="berry space" -Dblue="berry \"quotes\"" -Dblack="berry \\backslashes\\ "
20+
spark.space.10 -Dstraw="berry space" -Dblue="berry \"quotes\"" -Dblack="berry \\backslashes\\ "
21+
spark.space.11 \
22+
-Dstraw="berry space" \
23+
-Dblue="berry \"quotes\"" \
24+
-Dblack="berry \\backslashes\\ " \
25+
-Dcherry=berry
26+
spark.space.12 -Dstraw="berry space" -Dblue="berry \"quotes\"" \
27+
-Dblack="berry \\backslashes\\ " -Dcherry=berry
28+
29+
# Equal sign delimiter
30+
spark.equal.1=-Dstraw=berry
31+
spark.equal.2=-Dstraw="berry"
32+
spark.equal.3=-Dstraw="berry again"
33+
spark.equal.4=-Dstraw="berry \"quote"
34+
spark.equal.5=-Dstraw="berry \\backslash"
35+
spark.equal.6=-Dstraw="berry \"quotes\" and \\backslashes\\ "
36+
spark.equal.7=-Dstraw=berry -Dblue=berry -Dblack=berry
37+
spark.equal.8=-Dstraw="berry space" -Dblue="berry" -Dblack=berry
38+
spark.equal.9=-Dstraw="berry space" -Dblue="berry \"quotes\"" -Dblack="berry \\backslashes\\ "
39+
spark.equal.10 = -Dstraw="berry space" -Dblue="berry \"quotes\"" -Dblack="berry \\backslashes\\ "
40+
spark.equal.11=\
41+
-Dstraw="berry space" \
42+
-Dblue="berry \"quotes\"" \
43+
-Dblack="berry \\backslashes\\ " \
44+
-Dcherry=berry
45+
spark.equal.12=-Dstraw="berry space" -Dblue="berry \"quotes\"" \
46+
-Dblack="berry \\backslashes\\ " -Dcherry=berry
47+
48+
# Colon delimiter
49+
spark.colon.1:-Dstraw=berry
50+
spark.colon.2:-Dstraw="berry"
51+
spark.colon.3:-Dstraw="berry again"
52+
spark.colon.4:-Dstraw="berry \"quote"
53+
spark.colon.5:-Dstraw="berry \\backslash"
54+
spark.colon.6:-Dstraw="berry \"quotes\" and \\backslashes\\ "
55+
spark.colon.7:-Dstraw=berry -Dblue=berry -Dblack=berry
56+
spark.colon.8:-Dstraw="berry space" -Dblue="berry" -Dblack=berry
57+
spark.colon.9:-Dstraw="berry space" -Dblue="berry \"quotes\"" -Dblack="berry \\backslashes\\ "
58+
spark.colon.10 : -Dstraw="berry space" -Dblue="berry \"quotes\"" -Dblack="berry \\backslashes\\ "
59+
spark.colon.11:\
60+
-Dstraw="berry space" \
61+
-Dblue="berry \"quotes\"" \
62+
-Dblack="berry \\backslashes\\ " \
63+
-Dcherry=berry
64+
spark.colon.12:-Dstraw="berry space" -Dblue="berry \"quotes\"" \
65+
-Dblack="berry \\backslashes\\ " -Dcherry=berry
66+

‎bin/test.sh‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
#
18+
19+
SPARK_HOME="$(cd `dirname $0`/..; pwd)"
20+
PROPERTIES_FILE="$SPARK_HOME/bin/test.conf"
21+
22+
# Load utility functions
23+
. "$SPARK_HOME/bin/utils.sh"
24+
25+
tests_failed=0
26+
27+
# Test parse_java_property. This takes in three parameters, the name of the config,
28+
# the expected value, and whether or not to ignore whitespace (e.g. for multi-line).
29+
test_parse_java_property() {
30+
key="$1"
31+
expected_value="$2"
32+
ignore_whitespace="$3"
33+
parse_java_property "$key"
34+
actual_value="$JAVA_PROPERTY_VALUE"
35+
echo " $key -> $actual_value"
36+
# Ignore whitespace for multi-line arguments
37+
if [[ -n "$ignore_whitespace" ]]; then
38+
expected_value=$(echo "$expected_value" | sed "s/[[:space:]]//g")
39+
actual_value=$(echo "$actual_value" | sed "s/[[:space:]]//g")
40+
fi
41+
if [[ "$actual_value" != "$expected_value" ]]; then
42+
echo " XXXXX TEST FAILED XXXXX"
43+
echo " expected: $expected_value"
44+
echo " actual: $actual_value"
45+
tests_failed=1
46+
fi
47+
}
48+
49+
# Test split_java_options. This takes in three or more parameters, the name of the config,
50+
# the expected number of java options, and values of the java options themselves.
51+
test_split_java_options() {
52+
key="$1"
53+
expected_size="$2"
54+
expected_values=("${@:3}")
55+
parse_java_property "$key"
56+
echo " $JAVA_PROPERTY_VALUE"
57+
split_java_options "$JAVA_PROPERTY_VALUE"
58+
if [[ "$expected_size" != "${#SPLIT_JAVA_OPTS[@]}" ]]; then
59+
echo " XXXXX TEST FAILED XXXXX"
60+
echo " expected size: $expected_size"
61+
echo " actual size: ${#SPLIT_JAVA_OPTS[@]}"
62+
fi
63+
for i in $(seq 0 $((expected_size - 1))); do
64+
expected_value="${expected_values[$i]}"
65+
actual_value="${SPLIT_JAVA_OPTS[$i]}"
66+
echo " -> $actual_value"
67+
if [[ "$expected_value" != "$actual_value" ]]; then
68+
echo " XXXXX TEST FAILED (key $key) XXXXX"
69+
echo " expected value: $expected_value"
70+
echo " actual value: $actual_value"
71+
tests_failed=1
72+
break
73+
fi
74+
done
75+
}
76+
77+
# Test split_java_options. This takes in three or more parameters, the name of the config,
78+
# the expected number of java options, and values of the java options themselves.
79+
test_quote_java_property() {
80+
key="$1"
81+
expected_size="$2"
82+
expected_values=("${@:3}")
83+
parse_java_property "$key"
84+
split_java_options "$JAVA_PROPERTY_VALUE"
85+
quote_java_property "${SPLIT_JAVA_OPTS[@]}"
86+
echo " $JAVA_PROPERTY_VALUE"
87+
for i in $(seq 0 $((expected_size - 1))); do
88+
expected_value="${expected_values[$i]}"
89+
actual_value="${QUOTED_JAVA_OPTS[$i]}"
90+
echo " -> $actual_value"
91+
if [[ "$expected_value" != "$actual_value" ]]; then
92+
echo " XXXXX TEST FAILED (key $key) XXXXX"
93+
echo " expected value: $expected_value"
94+
echo " actual value: $actual_value"
95+
tests_failed=1
96+
break
97+
fi
98+
done
99+
}
100+
101+
# Test parse_java_property. This should read the literal value as written in the conf file.
102+
echo "--- Testing parse_java_property ---"
103+
delimiters=("space" "equal" "colon")
104+
test_parse_java_property "does.not.exist" ""
105+
for delimiter in "${delimiters[@]}"; do
106+
test_parse_java_property "spark.$delimiter.1" "-Dstraw=berry"
107+
test_parse_java_property "spark.$delimiter.2" "-Dstraw=\"berry\""
108+
test_parse_java_property "spark.$delimiter.3" "-Dstraw=\"berry again\""
109+
test_parse_java_property "spark.$delimiter.4" "-Dstraw=\"berry \\\"quote\""
110+
test_parse_java_property "spark.$delimiter.5" "-Dstraw=\"berry \\\\backslash\""
111+
test_parse_java_property "spark.$delimiter.6" "-Dstraw=\"berry \\\"quotes\\\" and \\\\backslashes\\\\ \""
112+
test_parse_java_property "spark.$delimiter.7" "-Dstraw=berry -Dblue=berry -Dblack=berry"
113+
test_parse_java_property "spark.$delimiter.8" "-Dstraw=\"berry space\" -Dblue=\"berry\" -Dblack=berry"
114+
test_parse_java_property "spark.$delimiter.9" \
115+
"-Dstraw=\"berry space\" -Dblue=\"berry \\\"quotes\\\"\" -Dblack=\"berry \\\\backslashes\\\\ \""
116+
test_parse_java_property "spark.$delimiter.10" \
117+
"-Dstraw=\"berry space\" -Dblue=\"berry \\\"quotes\\\"\" -Dblack=\"berry \\\\backslashes\\\\ \"" IGNORE_WHITESPACE
118+
test_parse_java_property "spark.$delimiter.11" \
119+
"-Dstraw=\"berry space\" -Dblue=\"berry \\\"quotes\\\"\" -Dblack=\"berry \\\\backslashes\\\\ \" -Dcherry=berry" IGNORE_WHITESPACE
120+
test_parse_java_property "spark.$delimiter.12" \
121+
"-Dstraw=\"berry space\" -Dblue=\"berry \\\"quotes\\\"\" -Dblack=\"berry \\\\backslashes\\\\ \" -Dcherry=berry" IGNORE_WHITESPACE
122+
done
123+
echo
124+
125+
# Test split_java_options. Note that this relies on parse_java_property to work correctly.
126+
if [[ "$tests_failed" == 1 ]]; then
127+
echo "* WARNING: Tests for parse_java_property failed!"
128+
echo -e "This should also fail tests for split_java_options\n"
129+
fi
130+
echo "--- Testing split_java_options ---"
131+
test_split_java_options "spark.space.1" 1 "-Dstraw=berry"
132+
test_split_java_options "spark.space.2" 1 "-Dstraw=berry"
133+
test_split_java_options "spark.space.3" 1 "-Dstraw=berry again"
134+
test_split_java_options "spark.space.4" 1 "-Dstraw=berry \"quote"
135+
test_split_java_options "spark.space.5" 1 "-Dstraw=berry \\backslash"
136+
test_split_java_options "spark.space.6" 1 "-Dstraw=berry \"quotes\" and \\backslashes\\ "
137+
test_split_java_options "spark.space.7" 3 "-Dstraw=berry" "-Dblue=berry" "-Dblack=berry"
138+
test_split_java_options "spark.space.8" 3 "-Dstraw=berry space" "-Dblue=berry" "-Dblack=berry"
139+
test_split_java_options "spark.space.9" 3 \
140+
"-Dstraw=berry space" "-Dblue=berry \"quotes\"" "-Dblack=berry \\backslashes\\ "
141+
test_split_java_options "spark.space.10" 3 \
142+
"-Dstraw=berry space" "-Dblue=berry \"quotes\"" "-Dblack=berry \\backslashes\\ "
143+
test_split_java_options "spark.space.11" 4 \
144+
"-Dstraw=berry space" "-Dblue=berry \"quotes\"" "-Dblack=berry \\backslashes\\ " "-Dcherry=berry"
145+
test_split_java_options "spark.space.12" 4 \
146+
"-Dstraw=berry space" "-Dblue=berry \"quotes\"" "-Dblack=berry \\backslashes\\ " "-Dcherry=berry"
147+
echo
148+
149+
# Test quote_java_property. Note that this relies on split_java_options to work correctly.
150+
if [[ "$tests_failed" == 1 ]]; then
151+
echo "* WARNING: Tests for split_java_options failed!"
152+
echo -e "This should also fail tests for quote_java_property\n"
153+
fi
154+
echo "--- Testing quote_java_property ---"
155+
test_quote_java_property "spark.space.1" 1 "\"-Dstraw=berry\""
156+
test_quote_java_property "spark.space.2" 1 "\"-Dstraw=berry\""
157+
test_quote_java_property "spark.space.3" 1 "\"-Dstraw=berry again\""
158+
test_quote_java_property "spark.space.4" 1 "\"-Dstraw=berry \"quote\""
159+
test_quote_java_property "spark.space.5" 1 "\"-Dstraw=berry \\backslash\""
160+
test_quote_java_property "spark.space.6" 1 "\"-Dstraw=berry \"quotes\" and \\backslashes\\ \""
161+
test_quote_java_property "spark.space.7" 3 "\"-Dstraw=berry\"" "\"-Dblue=berry\"" "\"-Dblack=berry\""
162+
test_quote_java_property "spark.space.8" 3 "\"-Dstraw=berry space\"" "\"-Dblue=berry\"" "\"-Dblack=berry\""
163+
test_quote_java_property "spark.space.9" 3 \
164+
"\"-Dstraw=berry space\"" "\"-Dblue=berry \"quotes\"\"" "\"-Dblack=berry \\backslashes\\ \""
165+
test_quote_java_property "spark.space.10" 3 \
166+
"\"-Dstraw=berry space\"" "\"-Dblue=berry \"quotes\"\"" "\"-Dblack=berry \\backslashes\\ \""
167+
test_quote_java_property "spark.space.11" 4 \
168+
"\"-Dstraw=berry space\"" "\"-Dblue=berry \"quotes\"\"" "\"-Dblack=berry \\backslashes\\ \"" "\"-Dcherry=berry\""
169+
test_quote_java_property "spark.space.12" 4 \
170+
"\"-Dstraw=berry space\"" "\"-Dblue=berry \"quotes\"\"" "\"-Dblack=berry \\backslashes\\ \"" "\"-Dcherry=berry\""
171+
echo
172+
173+
# Final test result
174+
if [[ "$tests_failed" == 0 ]]; then
175+
echo "**********************"
176+
echo " TESTS PASS "
177+
echo "**********************"
178+
else
179+
echo "XXXXXXXXXXXXXXXXXXXXXXXX"
180+
echo "XXXXX TESTS FAILED XXXXX"
181+
echo "XXXXXXXXXXXXXXXXXXXXXXXX"
182+
exit 1
183+
fi
184+

0 commit comments

Comments
 (0)