浮雲雅築: [研究] hadoop-2.2.0-src.tar.gz 編譯研究(CentOS 6.5 x86

[研究] hadoop-2.2.0-src.tar.gz 編譯研究(CentOS 6.5 x86_64)

2014-02-07

請先切換成 root 後執行

[root@localhost ~]# su root
[root@localhost ~]# vi Hadoop-2.2.0_CentOS-6.5-x86_64-Compile.sh
把下面框中內容全部貼上
[root@localhost ~]# chmod +x Hadoop-2.2.0_CentOS-6.5-x86_64-Compile.sh
[root@localhost ~]# ./Hadoop-2.2.0_CentOS-6.5-x86_64-Compile.sh

注意：請勿把下面程式直接貼到命令提示 (例如：[root@localhost ~]# ) 下直接執行，不然 if then ... else if 這類不是單一行命令的程式會執行錯誤。

快速安裝程式內容如下

#!/bin/bash

echo -e "\033[31m"
echo -e "Program : Hadoop-2.2.0_CentOS-6.5-x86_64-Compile.sh "
echo -e "Hadoop 2.2.0 Compile Shell Script (CentOS 6.5 x86_64) "
echo -e "by Shau-Rong Lu 2014-02-07 "
echo -e "\033[0m"

cd /usr/local/src
#yum -y groupinstall “Development tools”
#yum -y install gcc gcc-c++ svn cmake git zlib zlib-devel openssl openssl-devel rsync java-1.7.0-openjdk.x86_64 java-1.7.0-openjdk-devel.x86_64 make wget
yum -y install gcc gcc-c++ svn cmake git zlib zlib-devel openssl openssl-devel rsync make wget

# echo "********** Install OpenJDK **********"

#cd /usr/local/src
#yum -y install java-1.7.0-openjdk.x86_64 java-1.7.0-openjdk-devel.x86_64

#echo 'export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk-1.7.0.51.x86_64' >> /etc/profile
#echo 'export PATH=$PATH:$JAVA_HOME/bin' >> /etc/profile
#echo 'export CLASSPATH=$JAVA_HOME/jre/lib/ext:$JAVA_HOME/lib/tools.jar' >> /etc/profile

#source /etc/profile

#java -version
#export | grep jdk

echo "********** Install Oracle JDK 7 Update 51 **********"

cd /usr/local/src
#[研究] Oracle Java 7 Update 51 快速安裝程式(CentOS 6.5 x64)
# http://shaurong.blogspot.tw/2014/01/oracle-java-7-update-51-centos-65-x64.html

if [ ! -s jdk-7u51-linux-x64.rpm ]; then
echo "Can not find /usr/local/jdk-7u51-linux-x64.rpm"
wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F" "http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.rpm" -O jdk-7u51-linux-x64.rpm
#exit
fi

rpm -ivh jdk-7u51-linux-x64.rpm
alternatives --install /usr/bin/java java /usr/java/jdk1.7.0_51/bin/java 100
alternatives --set java /usr/java/jdk1.7.0_51/bin/java

echo
echo 'export JAVA_HOME=/usr/java/jdk1.7.0_51' >> /etc/profile
echo 'export PATH=$PATH:$JAVA_HOME/bin' >> /etc/profile
echo 'export CLASSPATH=$JAVA_HOME/jre/lib/ext:$JAVA_HOME/lib/tools.jar' >> /etc/profile

source /etc/profile

java -version
export | grep jdk

echo "********** Install Apache Maven 3.0.5 **********"
cd /usr/local/src
if [ ! -s apache-maven-3.0.5-bin.tar.gz ]; then
wget http://ftp.tc.edu.tw/pub/Apache/maven/maven-3/3.0.5/binaries/apache-maven-3.0.5-bin.tar.gz
fi
tar zxvf apache-maven-3.0.5-bin.tar.gz -C /usr/local
ln -s /usr/local/apache-maven-3.0.5/bin/mvn /usr/bin/mvn

echo 'export MAVEN_HOME=/usr/local/apache-maven-3.0.5' >> /etc/profile
echo 'export PATH=$PATH:$MAVEN_HOME/bin' >> /etc/profile
source /etc/profile
#read -n 1 -p "Press Enter to continue..."

#echo "********** Install Apache Maven 3.1.1 **********"
#cd /usr/local/src
#if [ ! -s apache-maven-3.1.1-bin.tar.gz ]; then
# wget http://ftp.tc.edu.tw/pub/Apache/maven/maven-3/3.1.1/binaries/apache-maven-3.1.1-bin.tar.gz
#fi
#tar zxvf apache-maven-3.1.1-bin.tar.gz -C /usr/local
#ln -s /usr/local/apache-maven-3.1.1/bin/mvn /usr/bin/mvn

#echo 'export MAVEN_HOME=/usr/local/apache-maven-3.1.1' >> /etc/profile
#echo 'export PATH=$PATH:$MAVEN_HOME/bin' >> /etc/profile
#source /etc/profile
#read -n 1 -p "Press Enter to continue..."

echo "********** Install FindBugs 2.0.3 **********"
cd /usr/local/src
if [ ! -s findbugs-2.0.3.tar.gz ]; then
wget http://prdownloads.sourceforge.net/findbugs/findbugs-2.0.3.tar.gz?download
fi
tar zxvf findbugs-2.0.3.tar.gz -C /usr/local/
ln -s /usr/local/findbugs-2.0.3/bin/findbugs /usr/bin/findbugs
echo 'export FINDBUGS_HOME=/usr/local/findbugs-2.0.3' >> /etc/profile
echo 'export PATH=$PATH:$FINDBUGS_HOME/bin' >> /etc/profile
source /etc/profile
#read -n 1 -p "Press Enter to continue..."

echo "********** Install Protoc 2.5.0 **********"
cd /usr/local/src
if [ ! -s protobuf-2.5.0.tar.gz ]; then
wget https://protobuf.googlecode.com/files/protobuf-2.5.0.tar.gz
fi
tar zxvf protobuf-2.5.0.tar.gz -C /usr/local/src
cd /usr/local/src/protobuf-2.5.0
./configure
make
make install
ln -s /usr/local/bin/protoc /usr/bin/protoc
echo 'export PROTO_HOME=/usr/local/' >> /etc/profile
echo 'export PATH=$PATH:$PROTO_HOME/bin' >> /etc/profile
source /etc/profile
#read -n 1 -p "Press Enter to continue..."

echo "********** Compile Hadoop 2.2.0 **********"

cd /usr/local/src
if [ ! -s hadoop-2.2.0-src.tar.gz ]; then
wget http://ftp.twaren.net/Unix/Web/apache/hadoop/common/hadoop-2.2.0/hadoop-2.2.0.tar.gz
#wget http://ftp.mirror.tw/pub/apache/hadoop/common/hadoop-2.2.0/hadoop-2.2.0-src.tar.gz
#wget http://apache.stu.edu.tw/hadoop/common/hadoop-2.2.0/hadoop-2.2.0-src.tar.gz
fi
tar zxvf hadoop-2.2.0-src.tar.gz -C /usr/local/src
cd /usr/local/src/hadoop-2.2.0-src/
wget https://issues.apache.org/jira/secure/attachment/12614482/HADOOP-10110.patch
#patch -p0 /usr/local/src/hadoop-2.2.0-src/pom.xml < HADOOP-10110.patch
patch -p0 < HADOOP-10110.patch
#mvn clean
mvn package -Pdist,native -DskipTests -Dtar
#read -n 1 -p "Press Enter to continue..."

編譯失敗

[root@localhost ~]# mvn package -Pdist,native -DskipTests -Dtar

...(略)

[INFO] -------------------------------------------------------------
[ERROR] COMPILATION ERROR :
[INFO] -------------------------------------------------------------
[ERROR] /usr/local/src/hadoop-2.2.0-src/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java:[88,11] error: cannot access AbstractLifeCycle
[ERROR] class file for org.mortbay.component.AbstractLifeCycle not found
/usr/local/src/hadoop-2.2.0-src/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java:[96,29] error: cannot access LifeCycle
[ERROR] class file for org.mortbay.component.LifeCycle not found
/usr/local/src/hadoop-2.2.0-src/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java:[98,10] error: cannot find symbol
[ERROR] symbol: method start()
location: variable server of type Server
/usr/local/src/hadoop-2.2.0-src/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java:[104,12] error: cannot find symbol
[INFO] 4 errors
[INFO] -------------------------------------------------------------
[INFO] ------------------------------------------------------------------------
[INFO] Reactor Summary:
[INFO]
[INFO] Apache Hadoop Main ................................ SUCCESS [1:42.481s]
[INFO] Apache Hadoop Project POM ......................... SUCCESS [59.027s]
[INFO] Apache Hadoop Annotations ......................... SUCCESS [31.859s]
[INFO] Apache Hadoop Assemblies .......................... SUCCESS [0.308s]
[INFO] Apache Hadoop Project Dist POM .................... SUCCESS [29.214s]
[INFO] Apache Hadoop Maven Plugins ....................... SUCCESS [40.273s]
[INFO] Apache Hadoop Auth ................................ FAILURE [29.467s]
[INFO] Apache Hadoop Auth Examples ....................... SKIPPED
[INFO] Apache Hadoop Common .............................. SKIPPED
[INFO] Apache Hadoop NFS ................................. SKIPPED
[INFO] Apache Hadoop Common Project ...................... SKIPPED
[INFO] Apache Hadoop HDFS ................................ SKIPPED
[INFO] Apache Hadoop HttpFS .............................. SKIPPED
[INFO] Apache Hadoop HDFS BookKeeper Journal ............. SKIPPED
[INFO] Apache Hadoop HDFS-NFS ............................ SKIPPED
[INFO] Apache Hadoop HDFS Project ........................ SKIPPED
[INFO] hadoop-yarn ....................................... SKIPPED
[INFO] hadoop-yarn-api ................................... SKIPPED
[INFO] hadoop-yarn-common ................................ SKIPPED
[INFO] hadoop-yarn-server ................................ SKIPPED
[INFO] hadoop-yarn-server-common ......................... SKIPPED
[INFO] hadoop-yarn-server-nodemanager .................... SKIPPED
[INFO] hadoop-yarn-server-web-proxy ...................... SKIPPED
[INFO] hadoop-yarn-server-resourcemanager ................ SKIPPED
[INFO] hadoop-yarn-server-tests .......................... SKIPPED
[INFO] hadoop-yarn-client ................................ SKIPPED
[INFO] hadoop-yarn-applications .......................... SKIPPED
[INFO] hadoop-yarn-applications-distributedshell ......... SKIPPED
[INFO] hadoop-mapreduce-client ........................... SKIPPED
[INFO] hadoop-mapreduce-client-core ...................... SKIPPED
[INFO] hadoop-yarn-applications-unmanaged-am-launcher .... SKIPPED
[INFO] hadoop-yarn-site .................................. SKIPPED
[INFO] hadoop-yarn-project ............................... SKIPPED
[INFO] hadoop-mapreduce-client-common .................... SKIPPED
[INFO] hadoop-mapreduce-client-shuffle ................... SKIPPED
[INFO] hadoop-mapreduce-client-app ....................... SKIPPED
[INFO] hadoop-mapreduce-client-hs ........................ SKIPPED
[INFO] hadoop-mapreduce-client-jobclient ................. SKIPPED
[INFO] hadoop-mapreduce-client-hs-plugins ................ SKIPPED
[INFO] Apache Hadoop MapReduce Examples .................. SKIPPED
[INFO] hadoop-mapreduce .................................. SKIPPED
[INFO] Apache Hadoop MapReduce Streaming ................. SKIPPED
[INFO] Apache Hadoop Distributed Copy .................... SKIPPED
[INFO] Apache Hadoop Archives ............................ SKIPPED
[INFO] Apache Hadoop Rumen ............................... SKIPPED
[INFO] Apache Hadoop Gridmix ............................. SKIPPED
[INFO] Apache Hadoop Data Join ........................... SKIPPED
[INFO] Apache Hadoop Extras .............................. SKIPPED
[INFO] Apache Hadoop Pipes ............................... SKIPPED
[INFO] Apache Hadoop Tools Dist .......................... SKIPPED
[INFO] Apache Hadoop Tools ............................... SKIPPED
[INFO] Apache Hadoop Distribution ........................ SKIPPED
[INFO] Apache Hadoop Client .............................. SKIPPED
[INFO] Apache Hadoop Mini-Cluster ........................ SKIPPED
[INFO] ------------------------------------------------------------------------
[INFO] BUILD FAILURE
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 4:54.772s
[INFO] Finished at: Fri Feb 07 18:31:35 CST 2014
[INFO] Final Memory: 37M/90M
[INFO] ------------------------------------------------------------------------
[ERROR] Failed to execute goal org.apache.maven.plugins:maven-compiler-plugin:2.5.1:testCompile (default-testCompile) on project hadoop-auth: Compilation failure: Compilation failure:
[ERROR] /usr/local/src/hadoop-2.2.0-src/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java:[88,11] error: cannot access AbstractLifeCycle
[ERROR] class file for org.mortbay.component.AbstractLifeCycle not found
[ERROR] /usr/local/src/hadoop-2.2.0-src/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java:[96,29] error: cannot access LifeCycle
[ERROR] class file for org.mortbay.component.LifeCycle not found
[ERROR] /usr/local/src/hadoop-2.2.0-src/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java:[98,10] error: cannot find symbol
[ERROR] symbol: method start()
[ERROR] location: variable server of type Server
[ERROR] /usr/local/src/hadoop-2.2.0-src/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java:[104,12] error: cannot find symbol
[ERROR] -> [Help 1]
[ERROR]
[ERROR] To see the full stack trace of the errors, re-run Maven with the -e switch.
[ERROR] Re-run Maven using the -X switch to enable full debug logging.
[ERROR]
[ERROR] For more information about the errors and possible solutions, please read the following articles:
[ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/MojoFailureException
[ERROR]
[ERROR] After correcting the problems, you can resume the build with the command
[ERROR] mvn <goals> -rf :hadoop-auth
[root@localhost ~]#

執行下面進行修正，再次執行就會成功

cd /usr/local/src/hadoop-2.2.0-src/
wget https://issues.apache.org/jira/secure/attachment/12614482/HADOOP-10110.patch
#patch -p0 /usr/local/src/hadoop-2.2.0-src/pom.xml < HADOOP-10110.patch
patch -p0 < HADOOP-10110.patch
#mvn clean
mvn package -Pdist,native -DskipTests -Dtar

[root@localhost hadoop-2.2.0-src]# mvn package -Pdist,native -DskipTests -Dtar

...(略)

[INFO] Reactor Summary:
[INFO]
[INFO] Apache Hadoop Main ................................ SUCCESS [2.236s]
[INFO] Apache Hadoop Project POM ......................... SUCCESS [2.135s]
[INFO] Apache Hadoop Annotations ......................... SUCCESS [4.833s]
[INFO] Apache Hadoop Assemblies .......................... SUCCESS [0.408s]
[INFO] Apache Hadoop Project Dist POM .................... SUCCESS [2.677s]
[INFO] Apache Hadoop Maven Plugins ....................... SUCCESS [5.309s]
[INFO] Apache Hadoop Auth ................................ SUCCESS [15.899s]
[INFO] Apache Hadoop Auth Examples ....................... SUCCESS [10.794s]
[INFO] Apache Hadoop Common .............................. SUCCESS [4:02.632s]
[INFO] Apache Hadoop NFS ................................. SUCCESS [19.842s]
[INFO] Apache Hadoop Common Project ...................... SUCCESS [0.022s]
[INFO] Apache Hadoop HDFS ................................ SUCCESS [2:05.180s]
[INFO] Apache Hadoop HttpFS .............................. SUCCESS [19.988s]
[INFO] Apache Hadoop HDFS BookKeeper Journal ............. SUCCESS [28.977s]
[INFO] Apache Hadoop HDFS-NFS ............................ SUCCESS [3.564s]
[INFO] Apache Hadoop HDFS Project ........................ SUCCESS [0.020s]
[INFO] hadoop-yarn ....................................... SUCCESS [1:03.850s]
[INFO] hadoop-yarn-api ................................... SUCCESS [35.187s]
[INFO] hadoop-yarn-common ................................ SUCCESS [27.774s]
[INFO] hadoop-yarn-server ................................ SUCCESS [0.074s]
[INFO] hadoop-yarn-server-common ......................... SUCCESS [10.843s]
[INFO] hadoop-yarn-server-nodemanager .................... SUCCESS [13.725s]
[INFO] hadoop-yarn-server-web-proxy ...................... SUCCESS [2.897s]
[INFO] hadoop-yarn-server-resourcemanager ................ SUCCESS [16.199s]
[INFO] hadoop-yarn-server-tests .......................... SUCCESS [0.537s]
[INFO] hadoop-yarn-client ................................ SUCCESS [6.495s]
[INFO] hadoop-yarn-applications .......................... SUCCESS [0.074s]
[INFO] hadoop-yarn-applications-distributedshell ......... SUCCESS [3.381s]
[INFO] hadoop-mapreduce-client ........................... SUCCESS [0.106s]
[INFO] hadoop-mapreduce-client-core ...................... SUCCESS [24.286s]
[INFO] hadoop-yarn-applications-unmanaged-am-launcher .... SUCCESS [1.940s]
[INFO] hadoop-yarn-site .................................. SUCCESS [0.107s]
[INFO] hadoop-yarn-project ............................... SUCCESS [12.946s]
[INFO] hadoop-mapreduce-client-common .................... SUCCESS [18.948s]
[INFO] hadoop-mapreduce-client-shuffle ................... SUCCESS [2.901s]
[INFO] hadoop-mapreduce-client-app ....................... SUCCESS [9.847s]
[INFO] hadoop-mapreduce-client-hs ........................ SUCCESS [5.032s]
[INFO] hadoop-mapreduce-client-jobclient ................. SUCCESS [5.366s]
[INFO] hadoop-mapreduce-client-hs-plugins ................ SUCCESS [1.532s]
[INFO] Apache Hadoop MapReduce Examples .................. SUCCESS [6.694s]
[INFO] hadoop-mapreduce .................................. SUCCESS [2.391s]
[INFO] Apache Hadoop MapReduce Streaming ................. SUCCESS [4.830s]
[INFO] Apache Hadoop Distributed Copy .................... SUCCESS [19.646s]
[INFO] Apache Hadoop Archives ............................ SUCCESS [1.823s]
[INFO] Apache Hadoop Rumen ............................... SUCCESS [6.549s]
[INFO] Apache Hadoop Gridmix ............................. SUCCESS [5.531s]
[INFO] Apache Hadoop Data Join ........................... SUCCESS [3.148s]
[INFO] Apache Hadoop Extras .............................. SUCCESS [2.952s]
[INFO] Apache Hadoop Pipes ............................... SUCCESS [5.581s]
[INFO] Apache Hadoop Tools Dist .......................... SUCCESS [1.649s]
[INFO] Apache Hadoop Tools ............................... SUCCESS [0.020s]
[INFO] Apache Hadoop Distribution ........................ SUCCESS [17.393s]
[INFO] Apache Hadoop Client .............................. SUCCESS [5.377s]
[INFO] Apache Hadoop Mini-Cluster ........................ SUCCESS [0.080s]
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 13:54.292s
[INFO] Finished at: Fri Feb 07 19:03:17 CST 2014
[INFO] Final Memory: 85M/239M
[INFO] ------------------------------------------------------------------------
[root@localhost hadoop-2.2.0-src]#

******************************************************

顯示 HADOOP-10110.patch 內容

[root@localhost hadoop-2.2.0-src]# cat HADOOP-10110.patch

Index: hadoop-common-project/hadoop-auth/pom.xml

===================================================================

--- hadoop-common-project/hadoop-auth/pom.xml (revision 1543124)

+++ hadoop-common-project/hadoop-auth/pom.xml (working copy)

@@ -54,6 +54,11 @@

</dependency>

<groupId>org.mortbay.jetty</groupId>

+ <artifactId>jetty-util</artifactId>

+ <scope>test</scope>

+ </dependency>

+ <dependency>

+ <groupId>org.mortbay.jetty</groupId>

<artifactId>jetty</artifactId>

</dependency>

原來是修正 hadoop-2.2.0-src/hadoop-common-project/hadoop-auth/pom.xml 內容

(完)

浮雲雅築

2014年2月7日星期五

[研究] hadoop-2.2.0-src.tar.gz 編譯研究(CentOS 6.5 x86_64)

沒有留言:

張貼留言

2014年2月7日 星期五

[研究] hadoop-2.2.0-src.tar.gz 編譯研究(CentOS 6.5 x86_64)

沒有留言:

張貼留言

2014年2月7日星期五