Enable predicate pushdown for Parquet by using hadoop-lzo

.travis.yml

@@ -28,27 +28,43 @@ group: deprecated-2017Q3
 cache:
   directories:
     - $HOME/.m2/repository
+    - $HOME/.thrift
 
 services:
   - docker
 
+before_install:
+  - |
+    if [[ ! -e $HOME/.thrift/bin/thrift ]]; then
+      sudo apt-get install libboost-dev libboost-test-dev libboost-program-options-dev libboost-filesystem-dev libboost-thread-dev libevent-dev automake libtool flex bison pkg-config g++ libssl-dev
+      wget https://www.apache.org/dist/thrift/0.9.3/thrift-0.9.3.tar.gz
+      tar xfz thrift-0.9.3.tar.gz
+      cd thrift-0.9.3 && ./configure --without-cpp --without-c_glib --without-python --without-ruby --without-php --without-erlang --without-go --without-nodejs -q --prefix=$HOME/.thrift
+      sudo make install > thrift_make_install.log
+      cd ..
+    fi
+  - |
+    if [[ ! -e /usr/local/bin/thrift ]]; then
+      sudo ln -s $HOME/.thrift/bin/thrift /usr/local/bin/thrift
+    fi
+
 install:
   - ./mvnw -v
   - |
     if [[ -v TEST_SPECIFIC_MODULES ]]; then
-      ./mvnw install $MAVEN_FAST_INSTALL -pl $TEST_SPECIFIC_MODULES -am
+      ./mvnw install $MAVEN_FAST_INSTALL -P !twitter-modules -pl $TEST_SPECIFIC_MODULES -am
     fi
   - |
     if [[ -v TEST_OTHER_MODULES ]]; then
-      ./mvnw install $MAVEN_FAST_INSTALL -pl '!presto-docs,!presto-server,!presto-server-rpm'
+      ./mvnw install $MAVEN_FAST_INSTALL -P !twitter-modules  -pl '!presto-docs,!presto-server,!presto-server-rpm'
     fi
   - |
     if [[ -v PRODUCT_TESTS_BASIC_ENVIRONMENT || -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT ]]; then
-      ./mvnw install $MAVEN_FAST_INSTALL -pl '!presto-docs,!presto-server-rpm'
+      ./mvnw install $MAVEN_FAST_INSTALL -P !twitter-modules  -pl '!presto-docs,!presto-server-rpm'
     fi
   - |
     if [[ -v HIVE_TESTS ]]; then
-      ./mvnw install $MAVEN_FAST_INSTALL -pl presto-hive-hadoop2 -am
+      ./mvnw install $MAVEN_FAST_INSTALL -P !twitter-modules -pl presto-hive-hadoop2 -am
     fi
 
 before_script:
@@ -61,30 +77,40 @@ before_script:
 script:
   - |
     if [[ -v MAVEN_CHECKS ]]; then
-      ./mvnw install -DskipTests -B -T C1 -P ci
+      ./mvnw install -DskipTests -B -T C1 -P 'ci,!twitter-modules'
     fi
   - |
     if [[ -v TEST_SPECIFIC_MODULES ]]; then
-      ./mvnw test $MAVEN_SKIP_CHECKS_AND_DOCS -B -pl $TEST_SPECIFIC_MODULES $TEST_FLAGS
+      ./mvnw test $MAVEN_SKIP_CHECKS_AND_DOCS -P !twitter-modules -B -pl $TEST_SPECIFIC_MODULES $TEST_FLAGS
     fi
   - |
     if [[ -v TEST_OTHER_MODULES ]]; then
-      ./mvnw test $MAVEN_SKIP_CHECKS_AND_DOCS -B -pl $TEST_OTHER_MODULES
+      ./mvnw test $MAVEN_SKIP_CHECKS_AND_DOCS -P !twitter-modules -B -pl $TEST_OTHER_MODULES
     fi
   - |
     if [[ -v PRODUCT_TESTS_BASIC_ENVIRONMENT ]]; then
       presto-product-tests/bin/run_on_docker.sh \
-        multinode -x quarantine,big_query,storage_formats,profile_specific_tests,tpcds
+        multinode -x quarantine,big_query,storage_formats,profile_specific_tests,tpcds,cli,hive_connector
+    fi
+  - |
+    if [[ -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT ]]; then
+      presto-product-tests/bin/run_on_docker.sh \
+        singlenode -g hdfs_no_impersonation
+    fi
+  - |
+    if [[ -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT ]]; then
+      presto-product-tests/bin/run_on_docker.sh \
+        singlenode-hdfs-impersonation -g storage_formats,hdfs_impersonation
     fi
   - |
     if [[ -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT ]]; then
       presto-product-tests/bin/run_on_docker.sh \
-        singlenode-kerberos-hdfs-impersonation -g storage_formats,cli,hdfs_impersonation,authorization
+        singlenode-kerberos-hdfs-impersonation -g storage_formats,hdfs_impersonation,authorization
     fi
   - |
     if [[ -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT ]]; then
       presto-product-tests/bin/run_on_docker.sh \
-        singlenode-ldap -g ldap -x simba_jdbc
+        singlenode-ldap -g ldap -x simba_jdbc,ldap_cli
     fi
 # SQL server image sporadically hangs during the startup
 # TODO: Uncomment it once issue is fixed
@@ -97,7 +123,7 @@ script:
   - |
     if [[ -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT ]]; then
       presto-product-tests/bin/run_on_docker.sh \
-        multinode-tls -g smoke,cli,group-by,join,tls
+        multinode-tls -g smoke,group-by,join,tls
     fi
   - |
     if [[ -v HIVE_TESTS ]]; then
@@ -109,8 +135,9 @@ before_cache:
   - rm -rf $HOME/.m2/repository/com/facebook
 
 notifications:
-  slack:
-    secure: V5eyoGShxFoCcYJcp858vf/T6gC9KeMxL0C1EElcpZRcKBrIVZzvhek3HLHxZOxlghqnvNVsyDtU3u5orkEaAXeXj5c2dN+4XBsAB9oeN5MtQ0Z3VLAhZDqKIW1LzcXrq4DpzM0PkGhjfjum/P94/qFYk0UckPtB6a341AuYRo8=
+  hipchat:
+    rooms:
+      secure: peNh1KxwlxIpFyb60S8AMvaJThgh1LsjE+Whf1rYkJalVd2wUrqBIoyDKVSueyHD01hQ06gT7rBV6Pu/QcBMR1a9BbMCjERfxLZFUAheuC2Rsb+p1c4dyvBcFUGacgW7XWKCaVYGDGxuUvb0I3Z8cR6KxhK2xi88tHiqBGVGV2yI6zzOTpWVknMfFBtn+ONU1Ob2P6trclXaDyFd4MxubULri6CQdl35eQAq/VnmR3SZOgyVu3V30MGKwI3zhSli+3VqmW0JmaDGoHN6gznM1+VqABLgmIq0P+n+r5gdZWRCorq10NZCFMhVQ8U6rQHcL7sAniYJJsC/yRt6+pjyzIF4N+LSzZ7T+FLxQqT7k/1ukNgrujLDfTpn76Mo9eYTZmfAdzbm1QKJDACwr8Slqhq1jGzcrFMHunvXhVqjOs24R+JAHblY0O9PXvv7aR29GOQWDCvD7nV5QBUr8Xz5q7ozbLqHTI+yH02Jj4EaZ+azWYdRmnr9wDBxWMYBEgOdj4pII9b298XEDB72TxA3KpLTpdLxBTR+gIk/LjJqb/wb84xUv8gPXkaXccltGd5YI90c84cX8isbzNkAylzyfF2Eyueh0XbnMHfpFqBS7qaVM0/D+UxZkU0WNJ0x7G9XJvkiq49bZz2q1KLE4XuvVnTZSSjVSUAS8RtHfwUV33c=
 
 before_deploy:
   - mkdir /tmp/artifacts

README.md

@@ -1,4 +1,4 @@
-# Presto [![Build Status](https://travis-ci.org/prestodb/presto.svg?branch=master)](https://travis-ci.org/prestodb/presto)
+# Presto [![Build Status](https://travis-ci.org/twitter-forks/presto.svg?branch=twitter-master)](https://travis-ci.org/twitter-forks/presto)
 
 Presto is a distributed SQL query engine for big data.
 

pom.xml

@@ -10,7 +10,7 @@
 
     <groupId>com.facebook.presto</groupId>
     <artifactId>presto-root</artifactId>
-    <version>0.188</version>
+    <version>0.188-tw-0.43</version>
     <packaging>pom</packaging>
 
     <name>presto-root</name>
@@ -28,9 +28,9 @@
     </licenses>
 
     <scm>
-        <connection>scm:git:git://github.com/facebook/presto.git</connection>
-        <url>https://github.com/facebook/presto</url>
-        <tag>0.188</tag>
+        <connection>scm:git:git://github.com/twitter-forks/presto.git</connection>
+        <url>https://github.com/twitter-forks/presto</url>
+        <tag>0.188-tw-0.43</tag>
     </scm>
 
     <properties>
@@ -764,6 +764,12 @@
                 <version>3.6.1</version>
             </dependency>
 
+            <dependency>
+                <groupId>org.apache.commons</groupId>
+                <artifactId>commons-pool2</artifactId>
+                <version>2.4.2</version>
+            </dependency>
+
             <dependency>
                 <groupId>commons-codec</groupId>
                 <artifactId>commons-codec</artifactId>
@@ -846,6 +852,54 @@
                 <version>1.1.1.7</version>
             </dependency>
 
+            <dependency>
+                <groupId>org.apache.curator</groupId>
+                <artifactId>curator-recipes</artifactId>
+                <version>4.0.0</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>org.apache.zookeeper</groupId>
+                        <artifactId>zookeeper</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+
+            <dependency>
+                <groupId>org.apache.curator</groupId>
+                <artifactId>curator-framework</artifactId>
+                <version>4.0.0</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>org.apache.zookeeper</groupId>
+                        <artifactId>zookeeper</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+
+            <dependency>
+                <groupId>org.apache.curator</groupId>
+                <artifactId>curator-client</artifactId>
+                <version>4.0.0</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>org.apache.zookeeper</groupId>
+                        <artifactId>zookeeper</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+
+            <dependency>
+                <groupId>org.apache.curator</groupId>
+                <artifactId>curator-test</artifactId>
+                <version>2.12.0</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>org.apache.zookeeper</groupId>
+                        <artifactId>zookeeper</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+
             <dependency>
                 <groupId>org.apache.zookeeper</groupId>
                 <artifactId>zookeeper</artifactId>
@@ -869,7 +923,7 @@
             <dependency>
                 <groupId>com.101tec</groupId>
                 <artifactId>zkclient</artifactId>
-                <version>0.8</version>
+                <version>0.10</version>
                 <exclusions>
                     <exclusion>
                         <artifactId>log4j</artifactId>
@@ -879,6 +933,10 @@
                         <groupId>org.slf4j</groupId>
                         <artifactId>slf4j-log4j12</artifactId>
                     </exclusion>
+                    <exclusion>
+                        <groupId>org.apache.zookeeper</groupId>
+                        <artifactId>zookeeper</artifactId>
+                    </exclusion>
                 </exclusions>
             </dependency>
 
@@ -978,6 +1036,11 @@
                 <version>3.1.4-1</version>
             </dependency>
 
+            <dependency>
+                <groupId>com.hadoop.gplcompression</groupId>
+                <artifactId>hadoop-lzo</artifactId>
+                <version>0.4.16</version>
+            </dependency>
             <!-- force newer version to be used for dependencies -->
             <dependency>
                 <groupId>org.javassist</groupId>
@@ -1257,4 +1320,18 @@
             </plugin>
         </plugins>
     </build>
+
+    <profiles>
+        <profile>
+            <id>twitter-modules</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+            <modules>
+                <module>presto-kafka07</module>
+                <module>twitter-eventlistener-plugin</module>
+                <module>presto-twitter-server</module>
+            </modules>
+        </profile>
+    </profiles>
 </project>

presto-accumulo/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.188</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-accumulo</artifactId>

presto-array/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.188</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-array</artifactId>

presto-atop/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.188</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-atop</artifactId>

presto-base-jdbc/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.188</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-base-jdbc</artifactId>

presto-benchmark-driver/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.188</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-benchmark-driver</artifactId>

presto-benchmark/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>presto-root</artifactId>
         <groupId>com.facebook.presto</groupId>
-        <version>0.188</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-benchmark</artifactId>

presto-benchto-benchmarks/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.188</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-benchto-benchmarks</artifactId>

presto-blackhole/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.188</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-blackhole</artifactId>

presto-bytecode/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.188</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-bytecode</artifactId>

presto-cassandra/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.188</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-cassandra</artifactId>

presto-cli/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.188</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-cli</artifactId>

presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java

@@ -17,6 +17,7 @@
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.net.HostAndPort;
+import com.sun.security.auth.module.UnixSystem;
 import io.airlift.airline.Option;
 import io.airlift.units.Duration;
 
@@ -71,15 +72,16 @@ public class ClientOptions
     @Option(name = "--keystore-password", title = "keystore password", description = "Keystore password")
     public String keystorePassword;
 
+    // Pick the user name for the logged in user.
+    // Do not let it be overridden by users.
+    public String user = new UnixSystem().getUsername();
+
     @Option(name = "--truststore-path", title = "truststore path", description = "Truststore path")
     public String truststorePath;
 
     @Option(name = "--truststore-password", title = "truststore password", description = "Truststore password")
     public String truststorePassword;
 
-    @Option(name = "--user", title = "user", description = "Username")
-    public String user = System.getProperty("user.name");
-
     @Option(name = "--password", title = "password", description = "Prompt for password")
     public boolean password;
 

presto-client/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.188</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-client</artifactId>