Skip to content

Commit 6041bea

Browse files
flowbehappyflowbehappy
committed
add vlt source code
1 parent d377023 commit 6041bea

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+6516
-16
lines changed

Diff for: ‎indexr-codegen/pom.xml

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<parent>
6+
<artifactId>indexr</artifactId>
7+
<groupId>io.indexr</groupId>
8+
<version>0.5.0</version>
9+
</parent>
10+
<modelVersion>4.0.0</modelVersion>
11+
12+
<artifactId>indexr-codegen</artifactId>
13+
14+
<name>${project.groupId}:${project.artifactId}</name>
15+
16+
<dependencies>
17+
<dependency>
18+
<groupId>io.indexr</groupId>
19+
<artifactId>indexr-segment</artifactId>
20+
<version>${project.version}</version>
21+
</dependency>
22+
</dependencies>
23+
24+
</project>

Diff for: ‎indexr-codegen/src/main/java/io/indexr/vlt/codec/pack/PackingClassGen.java

+400
Large diffs are not rendered by default.

Diff for: ‎indexr-hive/pom.xml

+5
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,11 @@
4646
<artifactId>indexr-segment</artifactId>
4747
<version>${project.version}</version>
4848
</dependency>
49+
<dependency>
50+
<groupId>io.indexr</groupId>
51+
<artifactId>indexr-vlt-segment</artifactId>
52+
<version>${project.version}</version>
53+
</dependency>
4954
<dependency>
5055
<groupId>org.apache.hive</groupId>
5156
<artifactId>hive-exec</artifactId>

Diff for: ‎indexr-server/pom.xml

+5-5
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,11 @@
4747
<artifactId>indexr-segment</artifactId>
4848
<version>${project.version}</version>
4949
</dependency>
50-
<!--<dependency>-->
51-
<!--<groupId>org.eclipse.jetty</groupId>-->
52-
<!--<artifactId>jetty-server</artifactId>-->
53-
<!--<version>9.3.11.v20160721</version>-->
54-
<!--</dependency>-->
50+
<dependency>
51+
<groupId>io.indexr</groupId>
52+
<artifactId>indexr-vlt-segment</artifactId>
53+
<version>${project.version}</version>
54+
</dependency>
5555
<dependency>
5656
<groupId>org.apache.kafka</groupId>
5757
<artifactId>kafka_2.10</artifactId>

Diff for: ‎indexr-vlt-segment/pom.xml

+78
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<parent>
6+
<artifactId>indexr</artifactId>
7+
<groupId>io.indexr</groupId>
8+
<version>0.5.0</version>
9+
</parent>
10+
<modelVersion>4.0.0</modelVersion>
11+
12+
<artifactId>indexr-vlt-segment</artifactId>
13+
14+
<name>${project.groupId}:${project.artifactId}</name>
15+
16+
<dependencies>
17+
<dependency>
18+
<groupId>io.indexr</groupId>
19+
<artifactId>indexr-segment</artifactId>
20+
<version>${project.version}</version>
21+
</dependency>
22+
<dependency>
23+
<groupId>io.indexr</groupId>
24+
<artifactId>indexr-segment</artifactId>
25+
<version>${project.version}</version>
26+
<type>test-jar</type>
27+
<scope>test</scope>
28+
</dependency>
29+
<dependency>
30+
<groupId>it.unimi.dsi</groupId>
31+
<artifactId>fastutil</artifactId>
32+
<version>6.5.9</version>
33+
</dependency>
34+
</dependencies>
35+
36+
37+
<build>
38+
<plugins>
39+
<plugin>
40+
<artifactId>maven-enforcer-plugin</artifactId>
41+
<version>1.4.1</version>
42+
</plugin>
43+
<plugin>
44+
<groupId>org.apache.maven.plugins</groupId>
45+
<artifactId>maven-jar-plugin</artifactId>
46+
<version>2.6</version>
47+
</plugin>
48+
<plugin>
49+
<groupId>org.codehaus.mojo</groupId>
50+
<artifactId>exec-maven-plugin</artifactId>
51+
<version>1.2.1</version>
52+
<dependencies>
53+
<dependency>
54+
<groupId>io.indexr</groupId>
55+
<artifactId>indexr-codegen</artifactId>
56+
<version>${project.version}</version>
57+
</dependency>
58+
</dependencies>
59+
<executions>
60+
<execution>
61+
<phase>generate-sources</phase>
62+
<goals>
63+
<goal>java</goal>
64+
</goals>
65+
</execution>
66+
</executions>
67+
<configuration>
68+
<mainClass>io.indexr.vlt.codec.pack.PackingClassGen</mainClass>
69+
<includePluginDependencies>true</includePluginDependencies>
70+
<arguments>
71+
<argument>${basedir}/target/generated-sources</argument>
72+
</arguments>
73+
<sourceRoot>${basedir}/target/generated-sources</sourceRoot>
74+
</configuration>
75+
</plugin>
76+
</plugins>
77+
</build>
78+
</project>

Diff for: ‎indexr-vlt-segment/src/main/java/io/indexr/plugin/VLTPlugin.java

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package io.indexr.plugin;
2+
3+
import io.indexr.vlt.segment.VersionAdapter_VLT;
4+
5+
import io.indexr.segment.SegmentMode;
6+
7+
public class VLTPlugin {
8+
public static SegmentMode VLT = new SegmentMode(10001, "vlt", new VersionAdapter_VLT());
9+
10+
@InitPlugin
11+
public static void init() {
12+
SegmentMode.addMode(VLT);
13+
SegmentMode.setDefault(VLT);
14+
}
15+
}

Diff for: ‎indexr-vlt-segment/src/main/java/io/indexr/vlt/codec/Codec.java

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package io.indexr.vlt.codec;
2+
3+
public interface Codec extends Encoder, Decoder {
4+
5+
CodecType type();
6+
7+
static int outputBufferSize(CodecType type, int valCount, int dataSize) {
8+
int size;
9+
switch (type) {
10+
case SIMPLE_DICT:
11+
case DICT_COMPRESS:
12+
size = dataSize + (valCount << 2) + 1024;
13+
break;
14+
case DICT:
15+
//size = dataSize + (valCount << 2) + 1024 + (1 << 17) + (1 << 16);
16+
size = dataSize + (valCount << 2) + 1024;
17+
break;
18+
default:
19+
if (valCount <= 64) {
20+
size = dataSize << 1;
21+
} else {
22+
size = (int) (dataSize * 1.5);
23+
}
24+
}
25+
// minimal 1024 bytes.
26+
return Math.max(1 << 10, size);
27+
}
28+
}

Diff for: ‎indexr-vlt-segment/src/main/java/io/indexr/vlt/codec/CodecType.java

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package io.indexr.vlt.codec;
2+
3+
public enum CodecType {
4+
PLAIN(1),
5+
DELTA(2),
6+
RLE(3),
7+
SIMPLE_DICT(4),
8+
/** This codec is private used only */
9+
DICT(5),
10+
DICT_COMPRESS(6),
11+
//
12+
;
13+
14+
public final int id;
15+
16+
CodecType(int id) {
17+
this.id = id;
18+
}
19+
20+
public static CodecType fromId(int id) {
21+
for (CodecType t : values()) {
22+
if (t.id == id) {
23+
return t;
24+
}
25+
}
26+
throw new RuntimeException("Illegal codec id: " + id);
27+
}
28+
}

Diff for: ‎indexr-vlt-segment/src/main/java/io/indexr/vlt/codec/Decoder.java

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package io.indexr.vlt.codec;
2+
3+
// Returns how many bytes it consumed. Which should be equal to the return value of Encoder,
4+
// otherwise, you caught a bug.
5+
public interface Decoder {
6+
default int decodeInts(int valCount, long inputAddr, long outputAddr, int outputLimit) {
7+
throw new UnsupportedOperationException();
8+
}
9+
10+
default int decodeLongs(int valCount, long inputAddr, long outputAddr, int outputLimit) {
11+
throw new UnsupportedOperationException();
12+
}
13+
14+
default int decodeFloats(int valCount, long inputAddr, long outputAddr, int outputLimit) {
15+
throw new UnsupportedOperationException();
16+
}
17+
18+
default int decodeDoubles(int valCount, long inputAddr, long outputAddr, int ouputLimit) {
19+
throw new UnsupportedOperationException();
20+
}
21+
22+
default int decodeStrings(int valCount, long inputAddr, long outputAddr, int ouputLimit) {
23+
throw new UnsupportedOperationException();
24+
}
25+
}

Diff for: ‎indexr-vlt-segment/src/main/java/io/indexr/vlt/codec/DecoderItf.java

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package io.indexr.vlt.codec;
2+
3+
public interface DecoderItf {
4+
int decode(int valCount, long inputAddr, long outputAddr, int outputLimit);
5+
}

Diff for: ‎indexr-vlt-segment/src/main/java/io/indexr/vlt/codec/Encoder.java

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package io.indexr.vlt.codec;
2+
3+
import io.indexr.util.Wrapper;
4+
5+
/**
6+
* Returns the encoded data size.
7+
*/
8+
public interface Encoder {
9+
default int encodeInts(int valCount, long inputAddr, long outputAddr, int outputLimit) {
10+
return encodeInts(valCount, inputAddr, outputAddr, outputLimit, new Wrapper());
11+
}
12+
13+
default int encodeInts(int valCount, long inputAddr, long outputAddr, int outputLimit, Wrapper extraInfo) {
14+
throw new UnsupportedOperationException();
15+
}
16+
17+
default int encodeLongs(int valCount, long inputAddr, long outputAddr, int outputLimit) {
18+
return encodeLongs(valCount, inputAddr, outputAddr, outputLimit, new Wrapper());
19+
}
20+
21+
default int encodeLongs(int valCount, long inputAddr, long outputAddr, int outputLimit, Wrapper extraInfo) {
22+
throw new UnsupportedOperationException();
23+
}
24+
25+
default int encodeFloats(int valCount, long inputAddr, long outputAddr, int outputLimit) {
26+
return encodeFloats(valCount, inputAddr, outputAddr, outputLimit, new Wrapper());
27+
}
28+
29+
default int encodeFloats(int valCount, long inputAddr, long outputAddr, int outputLimit, Wrapper extraInfo) {
30+
throw new UnsupportedOperationException();
31+
}
32+
33+
default int encodeDoubles(int valCount, long inputAddr, long outputAddr, int outputLimit) {
34+
return encodeDoubles(valCount, inputAddr, outputAddr, outputLimit, new Wrapper());
35+
}
36+
37+
default int encodeDoubles(int valCount, long inputAddr, long outputAddr, int outputLimit, Wrapper extraInfo) {
38+
throw new UnsupportedOperationException();
39+
}
40+
41+
default int encodeStrings(int valCount, long inputAddr, long outputAddr, int outputLimit) {
42+
return encodeStrings(valCount, inputAddr, outputAddr, outputLimit, new Wrapper());
43+
}
44+
45+
default int encodeStrings(int valCount, long inputAddr, long outputAddr, int outputLimit, Wrapper extraInfo) {
46+
throw new UnsupportedOperationException();
47+
}
48+
49+
}

Diff for: ‎indexr-vlt-segment/src/main/java/io/indexr/vlt/codec/EncoderItf.java

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package io.indexr.vlt.codec;
2+
3+
import io.indexr.util.Wrapper;
4+
5+
public interface EncoderItf {
6+
int encode(int valCount, long inputAddr, long outputAddr, int outputLimit, Wrapper extraInfo);
7+
}

Diff for: ‎indexr-vlt-segment/src/main/java/io/indexr/vlt/codec/Encoding.java

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
package io.indexr.vlt.codec;
2+
3+
public enum Encoding {
4+
}

Diff for: ‎indexr-vlt-segment/src/main/java/io/indexr/vlt/codec/ErrorCode.java

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package io.indexr.vlt.codec;
2+
3+
public class ErrorCode {
4+
public static final int BUFFER_OVERFLOW = -1;
5+
public static final int DATA_CORRUPTED = -2;
6+
public static final int DATA_IMPROPER = -3;
7+
public static final int BUG = -100;
8+
9+
}

0 commit comments

Comments
 (0)