Skip to content

Commit 4f94001

Browse files
yiming187yiming187
committed
KYLIN-2428 Cleanup unnecessary shaded libraries for job/coprocessor/jdbc/server
1 parent a7126f0 commit 4f94001

File tree

52 files changed

+847
-898
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+847
-898
lines changed

‎assembly/pom.xml

+45-4
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,20 @@
1919

2020
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2121
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
22+
<modelVersion>4.0.0</modelVersion>
23+
24+
<artifactId>kylin-assembly</artifactId>
25+
<name>Apache Kylin - Assembly</name>
26+
2227
<parent>
2328
<artifactId>kylin</artifactId>
2429
<groupId>org.apache.kylin</groupId>
2530
<version>2.0.0-SNAPSHOT</version>
26-
2731
</parent>
28-
<modelVersion>4.0.0</modelVersion>
2932

30-
<artifactId>kylin-assembly</artifactId>
31-
<name>Apache Kylin - Assembly</name>
33+
<properties>
34+
<shadeBase>org.apache.kylin.job.shaded</shadeBase>
35+
</properties>
3236

3337
<dependencies>
3438
<dependency>
@@ -180,8 +184,45 @@
180184
<exclude>org.apache.zookeeper:*</exclude>
181185
<exclude>net.sf.ehcache:*</exclude>
182186
<exclude>org.apache.httpcomponents:*</exclude>
187+
<exclude>org.apache.commons:commons-email:*</exclude>
188+
<exclude>com.sun.mail:javax.mail:*</exclude>
189+
<exclude>javax.activation:activation:*</exclude>
183190
</excludes>
184191
</artifactSet>
192+
<relocations>
193+
<relocation>
194+
<pattern>com.fasterxml.jackson</pattern>
195+
<shadedPattern>${shadeBase}.com.fasterxml.jackson</shadedPattern>
196+
</relocation>
197+
<relocation>
198+
<pattern>com.tdunning</pattern>
199+
<shadedPattern>${shadeBase}.com.tdunning</shadedPattern>
200+
</relocation>
201+
<relocation>
202+
<pattern>com.fasterxml.jackson</pattern>
203+
<shadedPattern>${shadeBase}.com.fasterxml.jackson</shadedPattern>
204+
</relocation>
205+
<relocation>
206+
<pattern>org.apache.commons.codec</pattern>
207+
<shadedPattern>${shadeBase}.org.apache.commons.codec</shadedPattern>
208+
</relocation>
209+
<relocation>
210+
<pattern>org.apache.commons.collections</pattern>
211+
<shadedPattern>${shadeBase}.org.apache.commons.collections</shadedPattern>
212+
</relocation>
213+
<relocation>
214+
<pattern>org.apache.commons.lang3</pattern>
215+
<shadedPattern>${shadeBase}.org.apache.commons.lang3</shadedPattern>
216+
</relocation>
217+
<relocation>
218+
<pattern>org.apache.commons.logging</pattern>
219+
<shadedPattern>${shadeBase}.org.apache.commons.logging</shadedPattern>
220+
</relocation>
221+
<relocation>
222+
<pattern>org.roaringbitmap</pattern>
223+
<shadedPattern>${shadeBase}.org.roaringbitmap</shadedPattern>
224+
</relocation>
225+
</relocations>
185226
<filters>
186227
<filter>
187228
<artifact>*:*</artifact>

‎atopcalcite/pom.xml

+1
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
</exclusion>
4444
</exclusions>
4545
</dependency>
46+
<!-- It should be avatica(the shaded one), not avatica-core, since the inconsistency protobuf dependency with Hadoop -->
4647
<dependency>
4748
<groupId>org.apache.calcite.avatica</groupId>
4849
<artifactId>avatica</artifactId>

‎build/bin/get-properties.sh

+1-2
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ then
2525
exit -1
2626
fi
2727

28-
job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
2928
tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
30-
result=`java -cp $job_jar:$tool_jar org.apache.kylin.tool.KylinConfigCLI $1 2>/dev/null`
29+
result=`java -cp $tool_jar -Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-log4j.properties org.apache.kylin.tool.KylinConfigCLI $1 2>/dev/null`
3130
echo "$result"

‎build/bin/kylin.sh

+5-3
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ function retrieveDependency() {
3838
source ${dir}/setenv.sh
3939
fi
4040

41-
export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
41+
export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
4242
export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
4343
if [ -n "$KAFKA_HOME" ]
4444
then
@@ -103,7 +103,7 @@ then
103103
# KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh
104104
hbase ${KYLIN_EXTRA_START_OPTS} \
105105
-Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
106-
-Dlog4j.configuration=kylin-server-log4j.properties \
106+
-Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-server-log4j.properties \
107107
-Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \
108108
-Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
109109
-Djava.endorsed.dirs=${tomcat_root}/endorsed \
@@ -165,8 +165,10 @@ then
165165
then source ${dir}/setenv-tool.sh
166166
fi
167167

168+
hbase_original=${HBASE_CLASSPATH}
169+
export HBASE_CLASSPATH=${hbase_original}:${KYLIN_HOME}/tool/*
168170
exec hbase ${KYLIN_EXTRA_START_OPTS} -Dkylin.hive.dependency=${hive_dependency} -Dkylin.hbase.dependency=${hbase_dependency} -Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-log4j.properties "$@"
169-
171+
export HBASE_CLASSPATH=${hbase_original}
170172
else
171173
quit "Usage: 'kylin.sh [-v] start' or 'kylin.sh [-v] stop'"
172174
fi

‎build/conf/kylin-tools-log4j.properties

+5-4
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,12 @@
2525
# It's called kylin-tools-log4j.properties so that it won't distract users from the other more important log4j config file: kylin-server-log4j.properties
2626
# enable this by -Dlog4j.configuration=kylin-tools-log4j.properties
2727

28-
log4j.rootLogger=INFO,stdout
28+
log4j.rootLogger=INFO,stderr
2929

30-
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
31-
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
32-
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t %c{1}:%L]: %m%n
30+
log4j.appender.stderr=org.apache.log4j.ConsoleAppender
31+
log4j.appender.stderr.Target=System.err
32+
log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
33+
log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} %-5p [%t %c{1}:%L]: %m%n
3334

3435
#log4j.logger.org.apache.hadoop=ERROR
3536
log4j.logger.org.apache.kylin=DEBUG

‎build/script/prepare_libs.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ mkdir build/lib build/tool
3333
cp assembly/target/kylin-assembly-${version}-job.jar build/lib/kylin-job-${version}.jar
3434
cp storage-hbase/target/kylin-storage-hbase-${version}-coprocessor.jar build/lib/kylin-coprocessor-${version}.jar
3535
cp jdbc/target/kylin-jdbc-${version}.jar build/lib/kylin-jdbc-${version}.jar
36-
cp tool/target/kylin-tool-${version}-assembly.jar build/tool/kylin-tool-${version}.jar
36+
cp tool-assembly/target/kylin-tool-assembly-${version}-assembly.jar build/tool/kylin-tool-${version}.jar
3737

3838
# Copied file becomes 000 for some env (e.g. my Cygwin)
3939
chmod 644 build/lib/kylin-job-${version}.jar

‎core-common/pom.xml

+4-18
Original file line numberDiff line numberDiff line change
@@ -34,26 +34,10 @@
3434

3535
<dependencies>
3636
<!-- Basic Utilities -->
37-
<dependency>
38-
<groupId>com.fasterxml.jackson.core</groupId>
39-
<artifactId>jackson-databind</artifactId>
40-
</dependency>
41-
<dependency>
42-
<groupId>commons-lang</groupId>
43-
<artifactId>commons-lang</artifactId>
44-
</dependency>
4537
<dependency>
4638
<groupId>org.apache.commons</groupId>
4739
<artifactId>commons-lang3</artifactId>
4840
</dependency>
49-
<dependency>
50-
<groupId>commons-io</groupId>
51-
<artifactId>commons-io</artifactId>
52-
</dependency>
53-
<dependency>
54-
<groupId>commons-cli</groupId>
55-
<artifactId>commons-cli</artifactId>
56-
</dependency>
5741
<dependency>
5842
<groupId>org.apache.commons</groupId>
5943
<artifactId>commons-email</artifactId>
@@ -63,9 +47,11 @@
6347
<artifactId>httpclient</artifactId>
6448
</dependency>
6549
<dependency>
66-
<groupId>com.google.guava</groupId>
67-
<artifactId>guava</artifactId>
50+
<groupId>com.fasterxml.jackson.core</groupId>
51+
<artifactId>jackson-databind</artifactId>
6852
</dependency>
53+
54+
<!-- Provided -->
6955
<dependency>
7056
<groupId>com.google.code.findbugs</groupId>
7157
<artifactId>jsr305</artifactId>

‎core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java

+50
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,12 @@
1919
package org.apache.kylin.common.util;
2020

2121
import java.io.File;
22+
import java.io.IOException;
2223
import java.lang.reflect.Method;
2324
import java.net.URL;
2425
import java.net.URLClassLoader;
26+
import java.net.URLDecoder;
27+
import java.util.Enumeration;
2528
import java.util.HashMap;
2629
import java.util.Map;
2730
import java.util.WeakHashMap;
@@ -89,4 +92,51 @@ public static Object newInstance(String clz) {
8992
}
9093
}
9194

95+
public static String findContainingJar(Class<?> clazz) {
96+
return findContainingJar(clazz, null);
97+
}
98+
99+
/**
100+
* Load the first jar library contains clazz with preferJarKeyword matched. If preferJarKeyword is null, just load the
101+
* jar likes Hadoop Commons' ClassUtil
102+
* @param clazz
103+
* @param preferJarKeyWord
104+
* @return
105+
*/
106+
public static String findContainingJar(Class<?> clazz, String preferJarKeyWord) {
107+
ClassLoader loader = clazz.getClassLoader();
108+
String classFile = clazz.getName().replaceAll("\\.", "/") + ".class";
109+
110+
try {
111+
Enumeration e = loader.getResources(classFile);
112+
113+
URL url = null;
114+
do {
115+
if (!e.hasMoreElements()) {
116+
if (url == null)
117+
return null;
118+
else
119+
break;
120+
}
121+
122+
url = (URL) e.nextElement();
123+
if (!"jar".equals(url.getProtocol()))
124+
break;
125+
if (preferJarKeyWord != null && url.getPath().indexOf(preferJarKeyWord) != -1)
126+
break;
127+
if (preferJarKeyWord == null)
128+
break;
129+
} while (true);
130+
131+
String toReturn = url.getPath();
132+
if (toReturn.startsWith("file:")) {
133+
toReturn = toReturn.substring("file:".length());
134+
}
135+
136+
toReturn = URLDecoder.decode(toReturn, "UTF-8");
137+
return toReturn.replaceAll("!.*$", "");
138+
} catch (IOException var6) {
139+
throw new RuntimeException(var6);
140+
}
141+
}
92142
}

‎core-common/src/test/java/org/apache/kylin/common/util/ClassUtilTest.java

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.kylin.common.util;
20+
21+
import org.junit.Assert;
22+
import org.junit.Test;
23+
24+
public class ClassUtilTest {
25+
26+
@Test
27+
public void testFindContainingJar() throws ClassNotFoundException {
28+
Assert.assertTrue(ClassUtil.findContainingJar(Class.forName("org.apache.commons.beanutils.BeanUtils")).contains("commons-beanutils"));
29+
Assert.assertTrue(ClassUtil.findContainingJar(Class.forName("org.apache.commons.beanutils.BeanUtils"), "core").contains("commons-beanutils-core"));
30+
}
31+
32+
}

‎core-cube/pom.xml

+16-2
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,23 @@
4848
<groupId>commons-collections</groupId>
4949
<artifactId>commons-collections</artifactId>
5050
</dependency>
51+
52+
<!-- Provided -->
53+
<dependency>
54+
<groupId>commons-lang</groupId>
55+
<artifactId>commons-lang</artifactId>
56+
</dependency>
57+
<dependency>
58+
<groupId>commons-io</groupId>
59+
<artifactId>commons-io</artifactId>
60+
</dependency>
61+
<dependency>
62+
<groupId>com.google.guava</groupId>
63+
<artifactId>guava</artifactId>
64+
</dependency>
5165
<dependency>
52-
<groupId>com.n3twork.druid</groupId>
53-
<artifactId>extendedset</artifactId>
66+
<groupId>com.google.code.findbugs</groupId>
67+
<artifactId>jsr305</artifactId>
5468
</dependency>
5569
<dependency>
5670
<groupId>com.esotericsoftware</groupId>

‎core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ public CannotFilterExtendedColumnException(TblColRef tblColRef) {
9797
}
9898
}
9999

100-
public enum DeriveType {
100+
public enum DeriveType implements java.io.Serializable{
101101
LOOKUP, PK_FK, EXTENDED_COLUMN
102102
}
103103

0 commit comments

Comments
 (0)