From cb1364c1068f3032e384d50e3e3acac72b496e02 Mon Sep 17 00:00:00 2001 From: Zoltan Ivanfi Date: Wed, 9 Jan 2019 15:54:12 +0100 Subject: [PATCH 1/7] PARQUET-1490: Add branch-specific Travis steps The script for the main branch makes sure that POM files in the master branch do not refer to SNAPSHOT versions. The possiblity of script for feature branches will allow building a SNAPSHOT version of parquet-mr and depending on it in the POM files. --- .travis.yml | 30 +------------------------ dev/travis-before-install-master.sh | 23 +++++++++++++++++++ dev/travis-before-install.sh | 34 +++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 29 deletions(-) create mode 100644 dev/travis-before-install-master.sh create mode 100644 dev/travis-before-install.sh diff --git a/.travis.yml b/.travis.yml index e4e623f03b..874435a7cd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,34 +1,6 @@ language: java before_install: - - date - - sudo apt-get update -qq - - sudo apt-get install build-essential - - sudo apt-get install pv - - date - - mkdir protobuf_install - - pushd protobuf_install - - wget https://github.com/google/protobuf/archive/v3.5.1.tar.gz -O protobuf-3.5.1.tar.gz - - tar xzf protobuf-3.5.1.tar.gz - - cd protobuf-3.5.1 - - sudo apt-get install autoconf automake libtool curl make g++ unzip - - ./autogen.sh - - ./configure - - make - - sudo make install - - sudo ldconfig - - protoc --version - - popd - - date - - pwd - - sudo apt-get install -qq libboost-dev libboost-test-dev libboost-program-options-dev libevent-dev automake libtool flex bison pkg-config g++ libssl-dev - - wget -nv http://archive.apache.org/dist/thrift/0.9.3/thrift-0.9.3.tar.gz - - tar zxf thrift-0.9.3.tar.gz - - cd thrift-0.9.3 - - chmod +x ./configure - - ./configure --disable-gen-erl --disable-gen-hs --without-ruby --without-haskell --without-erlang --without-php --without-nodejs - - sudo make install - - cd .. - - date + - bash dev/travis-before-install.sh env: - HADOOP_PROFILE=default TEST_CODECS=uncompressed,brotli diff --git a/dev/travis-before-install-master.sh b/dev/travis-before-install-master.sh new file mode 100644 index 0000000000..9197202f8d --- /dev/null +++ b/dev/travis-before-install-master.sh @@ -0,0 +1,23 @@ +shopt -s globstar +fail_the_build= +reduced_pom="$(tempfile)" +for pom in **/pom.xml +do + xmlstarlet ed -N pom='http://maven.apache.org/POM/4.0.0' \ + -d '/pom:project/pom:version|/pom:project/pom:parent/pom:version|//comment()' "$pom" > "$reduced_pom" + if grep -q SNAPSHOT "$reduced_pom" + then + if [[ ! "$fail_the_build" ]] + then + printf "Error: POM files in the master branch can not refer to SNAPSHOT versions.\n" + fail_the_build=YES + fi + printf "\nOffending POM file: %s\nOffending content:\n" "$pom" + xmlstarlet ed -d "//*[count((.|.//*)[contains(text(), 'SNAPSHOT')]) = 0]" "$reduced_pom" + fi +done +rm "$reduced_pom" +if [[ "$fail_the_build" ]] +then + exit 1 +fi diff --git a/dev/travis-before-install.sh b/dev/travis-before-install.sh new file mode 100644 index 0000000000..24d4c6e319 --- /dev/null +++ b/dev/travis-before-install.sh @@ -0,0 +1,34 @@ +set -e +date +sudo apt-get update -qq +sudo apt-get install -qq build-essential pv autoconf automake libtool curl make \ + g++ unzip libboost-dev libboost-test-dev libboost-program-options-dev \ + libevent-dev automake libtool flex bison pkg-config g++ libssl-dev xmlstarlet +date +mkdir protobuf_install +pushd protobuf_install +wget https://github.com/google/protobuf/archive/v3.5.1.tar.gz -O protobuf-3.5.1.tar.gz +tar xzf protobuf-3.5.1.tar.gz +cd protobuf-3.5.1 +./autogen.sh +./configure +make +sudo make install +sudo ldconfig +protoc --version +popd +date +pwd +wget -nv http://archive.apache.org/dist/thrift/0.9.3/thrift-0.9.3.tar.gz +tar zxf thrift-0.9.3.tar.gz +cd thrift-0.9.3 +chmod +x ./configure +./configure --disable-gen-erl --disable-gen-hs --without-ruby --without-haskell --without-erlang --without-php --without-nodejs +sudo make install +cd .. +branch_specific_script="dev/travis-before-install-${TRAVIS_BRANCH}.sh" +if [[ -e "$branch_specific_script" ]] +then + . "$branch_specific_script" +fi +date From cad2d83b0fc4eb29e602a8fb576b756f990605f6 Mon Sep 17 00:00:00 2001 From: Zoltan Ivanfi Date: Wed, 9 Jan 2019 17:54:02 +0100 Subject: [PATCH 2/7] Added comments. --- dev/travis-before-install-master.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dev/travis-before-install-master.sh b/dev/travis-before-install-master.sh index 9197202f8d..dad32d2ef5 100644 --- a/dev/travis-before-install-master.sh +++ b/dev/travis-before-install-master.sh @@ -1,8 +1,10 @@ -shopt -s globstar fail_the_build= reduced_pom="$(tempfile)" +shopt -s globstar # Enables ** to match files in subdirectories recursively for pom in **/pom.xml do + # Removes the project/version and project/parent/version elements, because + # those are allowed to have SNAPSHOT in them. Also removes comments. xmlstarlet ed -N pom='http://maven.apache.org/POM/4.0.0' \ -d '/pom:project/pom:version|/pom:project/pom:parent/pom:version|//comment()' "$pom" > "$reduced_pom" if grep -q SNAPSHOT "$reduced_pom" @@ -13,6 +15,9 @@ do fail_the_build=YES fi printf "\nOffending POM file: %s\nOffending content:\n" "$pom" + # Removes every element that does not have SNAPSHOT in it or its + # descendants. As a result, we get a skeleton of the POM file with only the + # offending parts. xmlstarlet ed -d "//*[count((.|.//*)[contains(text(), 'SNAPSHOT')]) = 0]" "$reduced_pom" fi done From d9bbbb407ef8e7e078606e2d5105ebcb3b283077 Mon Sep 17 00:00:00 2001 From: Zoltan Ivanfi Date: Wed, 9 Jan 2019 19:05:09 +0100 Subject: [PATCH 3/7] Added licenses. --- dev/travis-before-install-master.sh | 16 ++++++++++++++++ dev/travis-before-install.sh | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/dev/travis-before-install-master.sh b/dev/travis-before-install-master.sh index dad32d2ef5..b344de0652 100644 --- a/dev/travis-before-install-master.sh +++ b/dev/travis-before-install-master.sh @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. fail_the_build= reduced_pom="$(tempfile)" shopt -s globstar # Enables ** to match files in subdirectories recursively diff --git a/dev/travis-before-install.sh b/dev/travis-before-install.sh index 24d4c6e319..5f9efbcd6c 100644 --- a/dev/travis-before-install.sh +++ b/dev/travis-before-install.sh @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e date sudo apt-get update -qq From d7745c81adfed046c1f504a96f0ee95f428d1931 Mon Sep 17 00:00:00 2001 From: Zoltan Ivanfi Date: Thu, 10 Jan 2019 14:03:09 +0100 Subject: [PATCH 4/7] Added more comments, renamed files. --- .travis.yml | 2 +- dev/travis-before-install-master.sh | 44 ------------------------- dev/travis-before-install.sh | 50 ----------------------------- 3 files changed, 1 insertion(+), 95 deletions(-) delete mode 100644 dev/travis-before-install-master.sh delete mode 100644 dev/travis-before-install.sh diff --git a/.travis.yml b/.travis.yml index 874435a7cd..708e816c56 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: java before_install: - - bash dev/travis-before-install.sh + - bash dev/travis_before-install.sh env: - HADOOP_PROFILE=default TEST_CODECS=uncompressed,brotli diff --git a/dev/travis-before-install-master.sh b/dev/travis-before-install-master.sh deleted file mode 100644 index b344de0652..0000000000 --- a/dev/travis-before-install-master.sh +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -fail_the_build= -reduced_pom="$(tempfile)" -shopt -s globstar # Enables ** to match files in subdirectories recursively -for pom in **/pom.xml -do - # Removes the project/version and project/parent/version elements, because - # those are allowed to have SNAPSHOT in them. Also removes comments. - xmlstarlet ed -N pom='http://maven.apache.org/POM/4.0.0' \ - -d '/pom:project/pom:version|/pom:project/pom:parent/pom:version|//comment()' "$pom" > "$reduced_pom" - if grep -q SNAPSHOT "$reduced_pom" - then - if [[ ! "$fail_the_build" ]] - then - printf "Error: POM files in the master branch can not refer to SNAPSHOT versions.\n" - fail_the_build=YES - fi - printf "\nOffending POM file: %s\nOffending content:\n" "$pom" - # Removes every element that does not have SNAPSHOT in it or its - # descendants. As a result, we get a skeleton of the POM file with only the - # offending parts. - xmlstarlet ed -d "//*[count((.|.//*)[contains(text(), 'SNAPSHOT')]) = 0]" "$reduced_pom" - fi -done -rm "$reduced_pom" -if [[ "$fail_the_build" ]] -then - exit 1 -fi diff --git a/dev/travis-before-install.sh b/dev/travis-before-install.sh deleted file mode 100644 index 5f9efbcd6c..0000000000 --- a/dev/travis-before-install.sh +++ /dev/null @@ -1,50 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -set -e -date -sudo apt-get update -qq -sudo apt-get install -qq build-essential pv autoconf automake libtool curl make \ - g++ unzip libboost-dev libboost-test-dev libboost-program-options-dev \ - libevent-dev automake libtool flex bison pkg-config g++ libssl-dev xmlstarlet -date -mkdir protobuf_install -pushd protobuf_install -wget https://github.com/google/protobuf/archive/v3.5.1.tar.gz -O protobuf-3.5.1.tar.gz -tar xzf protobuf-3.5.1.tar.gz -cd protobuf-3.5.1 -./autogen.sh -./configure -make -sudo make install -sudo ldconfig -protoc --version -popd -date -pwd -wget -nv http://archive.apache.org/dist/thrift/0.9.3/thrift-0.9.3.tar.gz -tar zxf thrift-0.9.3.tar.gz -cd thrift-0.9.3 -chmod +x ./configure -./configure --disable-gen-erl --disable-gen-hs --without-ruby --without-haskell --without-erlang --without-php --without-nodejs -sudo make install -cd .. -branch_specific_script="dev/travis-before-install-${TRAVIS_BRANCH}.sh" -if [[ -e "$branch_specific_script" ]] -then - . "$branch_specific_script" -fi -date From a4c21190b4ea8479a52f971f33c3ec4873485771 Mon Sep 17 00:00:00 2001 From: Zoltan Ivanfi Date: Thu, 10 Jan 2019 14:13:45 +0100 Subject: [PATCH 5/7] Actually add the renamed files... --- dev/travis-before_install-master.sh | 50 ++++++++++++++++++++++++++ dev/travis-before_install.sh | 55 +++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 dev/travis-before_install-master.sh create mode 100644 dev/travis-before_install.sh diff --git a/dev/travis-before_install-master.sh b/dev/travis-before_install-master.sh new file mode 100644 index 0000000000..2c865ce30f --- /dev/null +++ b/dev/travis-before_install-master.sh @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################################################################ +# This is a branch-specific script that gets invoked at the end of +# travis-before_install.sh. It is run for the master branch only. +################################################################################ + +fail_the_build= +reduced_pom="$(tempfile)" +shopt -s globstar # Enables ** to match files in subdirectories recursively +for pom in **/pom.xml +do + # Removes the project/version and project/parent/version elements, because + # those are allowed to have SNAPSHOT in them. Also removes comments. + xmlstarlet ed -N pom='http://maven.apache.org/POM/4.0.0' \ + -d '/pom:project/pom:version|/pom:project/pom:parent/pom:version|//comment()' "$pom" > "$reduced_pom" + if grep -q SNAPSHOT "$reduced_pom" + then + if [[ ! "$fail_the_build" ]] + then + printf "Error: POM files in the master branch can not refer to SNAPSHOT versions.\n" + fail_the_build=YES + fi + printf "\nOffending POM file: %s\nOffending content:\n" "$pom" + # Removes every element that does not have SNAPSHOT in it or its + # descendants. As a result, we get a skeleton of the POM file with only the + # offending parts. + xmlstarlet ed -d "//*[count((.|.//*)[contains(text(), 'SNAPSHOT')]) = 0]" "$reduced_pom" + fi +done +rm "$reduced_pom" +if [[ "$fail_the_build" ]] +then + exit 1 +fi diff --git a/dev/travis-before_install.sh b/dev/travis-before_install.sh new file mode 100644 index 0000000000..f6cfdebd4e --- /dev/null +++ b/dev/travis-before_install.sh @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################################################################ +# This script gets invoked by .travis.yml in the before_install step +################################################################################ + +set -e +date +sudo apt-get update -qq +sudo apt-get install -qq build-essential pv autoconf automake libtool curl make \ + g++ unzip libboost-dev libboost-test-dev libboost-program-options-dev \ + libevent-dev automake libtool flex bison pkg-config g++ libssl-dev xmlstarlet +date +mkdir protobuf_install +pushd protobuf_install +wget https://github.com/google/protobuf/archive/v3.5.1.tar.gz -O protobuf-3.5.1.tar.gz +tar xzf protobuf-3.5.1.tar.gz +cd protobuf-3.5.1 +./autogen.sh +./configure +make +sudo make install +sudo ldconfig +protoc --version +popd +date +pwd +wget -nv http://archive.apache.org/dist/thrift/0.9.3/thrift-0.9.3.tar.gz +tar zxf thrift-0.9.3.tar.gz +cd thrift-0.9.3 +chmod +x ./configure +./configure --disable-gen-erl --disable-gen-hs --without-ruby --without-haskell --without-erlang --without-php --without-nodejs +sudo make install +cd .. +branch_specific_script="dev/travis-before-install-${TRAVIS_BRANCH}.sh" +if [[ -e "$branch_specific_script" ]] +then + . "$branch_specific_script" +fi +date From 26ecd71e132edc514957bf2bc625335dace13b33 Mon Sep 17 00:00:00 2001 From: Zoltan Ivanfi Date: Thu, 10 Jan 2019 14:14:30 +0100 Subject: [PATCH 6/7] Change reference to renamed file. --- dev/travis-before_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/travis-before_install.sh b/dev/travis-before_install.sh index f6cfdebd4e..6d0427972c 100644 --- a/dev/travis-before_install.sh +++ b/dev/travis-before_install.sh @@ -47,7 +47,7 @@ chmod +x ./configure ./configure --disable-gen-erl --disable-gen-hs --without-ruby --without-haskell --without-erlang --without-php --without-nodejs sudo make install cd .. -branch_specific_script="dev/travis-before-install-${TRAVIS_BRANCH}.sh" +branch_specific_script="dev/travis-before_install-${TRAVIS_BRANCH}.sh" if [[ -e "$branch_specific_script" ]] then . "$branch_specific_script" From e58328c1d467d2cf49bcce3cb9b6dd0d42adf239 Mon Sep 17 00:00:00 2001 From: Zoltan Ivanfi Date: Thu, 10 Jan 2019 15:14:15 +0100 Subject: [PATCH 7/7] Fix reference to renamed script. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 708e816c56..d33051f57f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: java before_install: - - bash dev/travis_before-install.sh + - bash dev/travis-before_install.sh env: - HADOOP_PROFILE=default TEST_CODECS=uncompressed,brotli