文档章节

Apache Hadoop编译

GordonNemo
 GordonNemo
发布于 06/14 10:46
字数 1762
阅读 6
收藏 0
点赞 0
评论 0

1.Hadoop源代码下载 apache官网上下载。

[root@hadoop001 sourcecode]# pwd

/opt/sourcecode

[root@hadoop001 sourcecode]# ll

total 33756

-rw-r--r--. 1 root root 34523353 Aug 20 12:14 hadoop-2.8.1-src.tar.gz

[root@hadoop001 sourcecode]# tar -xzvf hadoop-2.8.1-src.tar.gz

[root@hadoop001 sourcecode]# ll

total 33760

drwxr-xr-x. 17 root root     4096 Jun  2 14:13 hadoop-2.8.1-src

-rw-r--r--.  1 root root 34523353 Aug 20 12:14 hadoop-2.8.1-src.tar.gz

[root@hadoop001 sourcecode]# cd hadoop-2.8.1-src

[root@hadoop001 hadoop-2.8.1-src]# ll

total 224

-rw-rw-r--.  1 root root 15623 May 24 07:14 BUILDING.txt

drwxr-xr-x.  4 root root  4096 Aug 20 12:44 dev-support

drwxr-xr-x.  3 root root  4096 Aug 20 12:44 hadoop-assemblies

drwxr-xr-x.  3 root root  4096 Aug 20 12:44 hadoop-build-tools

drwxrwxr-x.  2 root root  4096 Aug 20 12:44 hadoop-client

drwxr-xr-x. 10 root root  4096 Aug 20 12:44 hadoop-common-project

drwxr-xr-x.  2 root root  4096 Aug 20 12:44 hadoop-dist

drwxr-xr-x.  8 root root  4096 Aug 20 12:44 hadoop-hdfs-project

drwxr-xr-x.  9 root root  4096 Aug 20 12:44 hadoop-mapreduce-project

drwxr-xr-x.  3 root root  4096 Aug 20 12:44 hadoop-maven-plugins

drwxr-xr-x.  2 root root  4096 Aug 20 12:44 hadoop-minicluster

drwxr-xr-x.  3 root root  4096 Aug 20 12:44 hadoop-project

drwxr-xr-x.  2 root root  4096 Aug 20 12:44 hadoop-project-dist

drwxr-xr-x. 18 root root  4096 Aug 20 12:44 hadoop-tools

drwxr-xr-x.  3 root root  4096 Aug 20 12:44 hadoop-yarn-project

-rw-rw-r--.  1 root root 99253 May 24 07:14 LICENSE.txt

-rw-rw-r--.  1 root root 15915 May 24 07:14 NOTICE.txt

drwxrwxr-x.  2 root root  4096 Jun  2 14:24 patchprocess

-rw-rw-r--.  1 root root 20477 May 29 06:36 pom.xml

-rw-r--r--.  1 root root  1366 May 20 13:30 README.txt

-rwxrwxr-x.  1 root root  1841 May 24 07:14 start-build-env.sh

[root@hadoop001 hadoop-2.8.1-src]#

[root@hadoop001 hadoop-2.8.1-src]# cat BUILDING.txt

Build instructions for Hadoop

 

----------------------------------------------------------------------------------

Requirements:

 

* Unix System

* JDK 1.7+

* Maven 3.0 or later

* Findbugs 1.3.9 (if running findbugs)

* ProtocolBuffer 2.5.0

* CMake 2.6 or newer (if compiling native code), must be 3.0 or newer on Mac

* Zlib devel (if compiling native code)

* openssl devel (if compiling native hadoop-pipes and to get the best HDFS encryption performance)

* Linux FUSE (Filesystem in Userspace) version 2.6 or above (if compiling fuse_dfs)

* Internet connection for first build (to fetch all Maven and Hadoop dependencies)

 

2.JAVA安装

[root@hadoop001 ~]# rz #上传jdk-8u45-linux-x64.gz

[root@hadoop001 ~]# mkdir -p /usr/java

[root@hadoop001 ~]# mv jdk-8u45-linux-x64.gz /usr/java

[root@hadoop001 ~]# cd /usr/java

[root@hadoop001 ~]# tar -xzvf jdk-8u45-linux-x64.gz

 

#修改用户和用户组

[root@hadoop002 java]# ll

total 169388

drwxr-xr-x. 8 uucp  143      4096 Apr 11  2015 jdk1.8.0_45

-rw-r--r--. 1 root root 173271626 Mar 16 15:25 jdk-8u45-linux-x64.gz

[root@hadoop002 java]# chown -R root:root jdk1.8.0_45

[root@hadoop002 java]# ll

total 169388

drwxr-xr-x. 8 root root      4096 Apr 11  2015 jdk1.8.0_45

-rw-r--r--. 1 root root 173271626 Mar 16 15:25 jdk-8u45-linux-x64.gz

[root@hadoop002 java]#

 

 

[root@hadoop001 java]# vi /etc/profile

export JAVA_HOME=/usr/java/jdk1.8.0_45

export PATH=$JAVA_HOME/bin:$PATH

 

[root@hadoop001 java]# source /etc/profile

[root@hadoop001 java]# which java

/usr/java/jdk1.8.0_45/bin/java

[root@hadoop001 java]# java -version

java version "1.8.0_45"

Java(TM) SE Runtime Environment (build 1.8.0_45-b14)

Java HotSpot(TM) 64-Bit Server VM (build 25.45-b02, mixed mode)

[root@hadoop001 java]#

 

3.Maven安装

[root@hadoop001 ~]# mkdir -p /opt/sourcecode /opt/software

 

[root@hadoop001 ~]# cd /opt/software/

[root@hadoop001 software]# rz

rz waiting to receive.

Starting zmodem transfer.  Press Ctrl+C to cancel.

Transferring apache-maven-3.3.9-bin.zip...

  100%    8415 KB    8415 KB/sec    00:00:01       0 Errors 

 

[root@hadoop001 software]#

[root@hadoop001 software]# ll

total 8432

-rw-r--r--. 1 root root 8617253 Aug 20 12:35 apache-maven-3.3.9-bin.zip

[root@hadoop001 software]# unzip apache-maven-3.3.9-bin.zip

 

 

[root@hadoop001 java]# vi /etc/profile

export MAVEN_HOME=/opt/software/apache-maven-3.3.9

export MAVEN_OPTS="-Xms256m -Xmx512m"

export PATH=$MAVEN_HOME/bin:$JAVA_HOME/bin:$PATH

 

[root@hadoop001 ~]# mvn -version

Apache Maven 3.3.9 (bb52d8502b132ec0a5a3f4c09453c07478323dc5; 2015-11-11T00:41:47+08:00)

Maven home: /opt/software/apache-maven-3.3.9

Java version: 1.8.0_45, vendor: Oracle Corporation

Java home: /usr/java/jdk1.8.0_45/jre

Default locale: en_US, platform encoding: UTF-8

OS name: "linux", version: "2.6.32-431.el6.x86_64", arch: "amd64", family: "unix"

[root@hadoop001 ~]#

 

 

4.protobuf安装

[root@hadoop001 software]# rz

rz waiting to receive.

Starting zmodem transfer.  Press Ctrl+C to cancel.

Transferring protobuf-2.5.0.tar.gz...

  100%    2345 KB    2345 KB/sec    00:00:01       0 Errors 

 

[root@hadoop001 software]# tar -xzvf protobuf-2.5.0.tar.gz

[root@hadoop001 software]# ll

total 10792

drwxr-xr-x.  6 root   root    4096 Nov 10  2015 apache-maven-3.3.9

-rw-r--r--.  1 root   root 8617253 Aug 20 12:35 apache-maven-3.3.9-bin.zip

drwxr-xr-x. 10 109965 5000    4096 Feb 27  2013 protobuf-2.5.0

-rw-r--r--.  1 root   root 2401901 Aug 20 13:03 protobuf-2.5.0.tar.gz

[root@hadoop001 software]# cd protobuf-2.5.0

[root@hadoop001 protobuf-2.5.0]# yum install -y gcc gcc-c++ make cmake   #已经安装cmake

[root@hadoop001 protobuf-2.5.0]# ./configure --prefix=/usr/local/protobuf

[root@hadoop001 protobuf-2.5.0]# make && make install

 

[root@hadoop001 java]# vi /etc/profile

export PROTOC_HOME=/usr/local/protobuf

export PATH=$PROTOC_HOME/bin:$FINDBUGS_HOME/bin:$MAVEN_HOME/bin:$JAVA_HOME/bin:$PATH

[root@hadoop001 protobuf-2.5.0]# source /etc/profile

 

[root@hadoop001 protobuf-2.5.0]# protoc --version

libprotoc 2.5.0

[root@hadoop001 protobuf-2.5.0]#

 

 

5.Findbugs安装

[root@hadoop001 software]# rz

rz waiting to receive.

Starting zmodem transfer.  Press Ctrl+C to cancel.

Transferring findbugs-1.3.9.zip...

  100%    7369 KB    7369 KB/sec    00:00:01       0 Errors 

 

[root@hadoop001 software]# unzip findbugs-1.3.9.zip

 

[root@hadoop002 software]# vi /etc/profile

export FINDBUGS_HOME=/opt/software/findbugs-1.3.9

export PATH=$FINDBUGS_HOME/bin:$MAVEN_HOME/bin:$JAVA_HOME/bin:$PATH

[root@hadoop001 software]#

[root@hadoop001 software]# source /etc/profile

[root@hadoop001 software]# findbugs -version

1.3.9

[root@hadoop001 software]#

 

6.其他依赖

yum install -y openssl openssl-devel svn ncurses-devel zlib-devel libtool

yum install -y snappy snappy-devel bzip2 bzip2-devel lzo lzo-devel lzop autoconf automake

 

7.编译

[root@hadoop001 sourcecode]# cd hadoop-2.8.1-src

[root@hadoop001 hadoop-2.8.1-src]# mvn clean package -Pdist,native -DskipTests -Dtar

[INFO] --- maven-site-plugin:3.5:attach-descriptor (attach-descriptor) @ hadoop-dist ---

[INFO] ------------------------------------------------------------------------

[INFO] Reactor Summary:

[INFO]

[INFO] Apache Hadoop Main ................................. SUCCESS [  7.770 s]

[INFO] Apache Hadoop Build Tools .......................... SUCCESS [  4.990 s]

[INFO] Apache Hadoop Project POM .......................... SUCCESS [  4.920 s]

[INFO] Apache Hadoop Annotations .......................... SUCCESS [ 12.936 s]

[INFO] Apache Hadoop Assemblies ........................... SUCCESS [  0.809 s]

[INFO] Apache Hadoop Project Dist POM ..................... SUCCESS [  5.013 s]

[INFO] Apache Hadoop Maven Plugins ........................ SUCCESS [ 17.617 s]

[INFO] Apache Hadoop MiniKDC .............................. SUCCESS [ 23.551 s]

[INFO] Apache Hadoop Auth ................................. SUCCESS [ 26.352 s]

[INFO] Apache Hadoop Auth Examples ........................ SUCCESS [ 10.305 s]

[INFO] Apache Hadoop Common ............................... SUCCESS [05:58 min]

[INFO] Apache Hadoop NFS .................................. SUCCESS [ 19.105 s]

[INFO] Apache Hadoop KMS .................................. SUCCESS [ 27.479 s]

[INFO] Apache Hadoop Common Project ....................... SUCCESS [  0.174 s]

[INFO] Apache Hadoop HDFS Client .......................... SUCCESS [01:32 min]

[INFO] Apache Hadoop HDFS ................................. SUCCESS [06:42 min]

[INFO] Apache Hadoop HDFS Native Client ................... SUCCESS [ 20.370 s]

[INFO] Apache Hadoop HttpFS ............................... SUCCESS [ 40.720 s]

[INFO] Apache Hadoop HDFS BookKeeper Journal .............. SUCCESS [ 21.298 s]

[INFO] Apache Hadoop HDFS-NFS ............................. SUCCESS [ 11.818 s]

[INFO] Apache Hadoop HDFS Project ......................... SUCCESS [  0.148 s]

[INFO] Apache Hadoop YARN ................................. SUCCESS [  0.192 s]

[INFO] Apache Hadoop YARN API ............................. SUCCESS [ 41.517 s]

[INFO] Apache Hadoop YARN Common .......................... SUCCESS [01:19 min]

[INFO] Apache Hadoop YARN Server .......................... SUCCESS [  0.192 s]

[INFO] Apache Hadoop YARN Server Common ................... SUCCESS [ 19.421 s]

[INFO] Apache Hadoop YARN NodeManager ..................... SUCCESS [ 42.398 s]

[INFO] Apache Hadoop YARN Web Proxy ....................... SUCCESS [  8.925 s]

[INFO] Apache Hadoop YARN ApplicationHistoryService ....... SUCCESS [ 16.120 s]

[INFO] Apache Hadoop YARN ResourceManager ................. SUCCESS [ 57.415 s]

[INFO] Apache Hadoop YARN Server Tests .................... SUCCESS [  3.869 s]

[INFO] Apache Hadoop YARN Client .......................... SUCCESS [ 14.325 s]

[INFO] Apache Hadoop YARN SharedCacheManager .............. SUCCESS [ 11.814 s]

[INFO] Apache Hadoop YARN Timeline Plugin Storage ......... SUCCESS [ 10.027 s]

[INFO] Apache Hadoop YARN Applications .................... SUCCESS [  0.276 s]

[INFO] Apache Hadoop YARN DistributedShell ................ SUCCESS [  8.333 s]

[INFO] Apache Hadoop YARN Unmanaged Am Launcher ........... SUCCESS [  5.473 s]

[INFO] Apache Hadoop YARN Site ............................ SUCCESS [  0.160 s]

[INFO] Apache Hadoop YARN Registry ........................ SUCCESS [ 13.204 s]

[INFO] Apache Hadoop YARN Project ......................... SUCCESS [  8.106 s]

[INFO] Apache Hadoop MapReduce Client ..................... SUCCESS [  0.514 s]

[INFO] Apache Hadoop MapReduce Core ....................... SUCCESS [01:09 min]

[INFO] Apache Hadoop MapReduce Common ..................... SUCCESS [ 40.479 s]

[INFO] Apache Hadoop MapReduce Shuffle .................... SUCCESS [ 10.304 s]

[INFO] Apache Hadoop MapReduce App ........................ SUCCESS [ 27.335 s]

[INFO] Apache Hadoop MapReduce HistoryServer .............. SUCCESS [ 19.910 s]

[INFO] Apache Hadoop MapReduce JobClient .................. SUCCESS [ 16.657 s]

[INFO] Apache Hadoop MapReduce HistoryServer Plugins ...... SUCCESS [  4.591 s]

[INFO] Apache Hadoop MapReduce Examples ................... SUCCESS [ 12.346 s]

[INFO] Apache Hadoop MapReduce ............................ SUCCESS [  5.966 s]

[INFO] Apache Hadoop MapReduce Streaming .................. SUCCESS [  7.940 s]

[INFO] Apache Hadoop Distributed Copy ..................... SUCCESS [ 15.245 s]

[INFO] Apache Hadoop Archives ............................. SUCCESS [  5.380 s]

[INFO] Apache Hadoop Archive Logs ......................... SUCCESS [  5.812 s]

[INFO] Apache Hadoop Rumen ................................ SUCCESS [ 11.785 s]

[INFO] Apache Hadoop Gridmix .............................. SUCCESS [  9.890 s]

[INFO] Apache Hadoop Data Join ............................ SUCCESS [  5.784 s]

[INFO] Apache Hadoop Ant Tasks ............................ SUCCESS [  3.254 s]

[INFO] Apache Hadoop Extras ............................... SUCCESS [  5.495 s]

[INFO] Apache Hadoop Pipes ................................ SUCCESS [ 10.630 s]

[INFO] Apache Hadoop OpenStack support .................... SUCCESS [ 11.234 s]

[INFO] Apache Hadoop Amazon Web Services support .......... SUCCESS [ 14.060 s]

[INFO] Apache Hadoop Azure support ........................ SUCCESS [ 10.535 s]

[INFO] Apache Hadoop Client ............................... SUCCESS [ 13.519 s]

[INFO] Apache Hadoop Mini-Cluster ......................... SUCCESS [  2.164 s]

[INFO] Apache Hadoop Scheduler Load Simulator ............. SUCCESS [ 10.405 s]

[INFO] Apache Hadoop Tools Dist ........................... SUCCESS [ 11.514 s]

[INFO] Apache Hadoop Azure Data Lake support .............. SUCCESS [  9.201 s]

[INFO] Apache Hadoop Tools ................................ SUCCESS [  0.129 s]

[INFO] Apache Hadoop Distribution ......................... SUCCESS [01:07 min]

[INFO] ------------------------------------------------------------------------

[INFO] BUILD SUCCESS

[INFO] ------------------------------------------------------------------------

[INFO] Total time: 31:41 min

[INFO] Finished at: 2017-12-10T11:55:28+08:00

[INFO] Final Memory: 166M/494M

[INFO] ------------------------------------------------------------------------

[root@rzdatahadoop001 hadoop-2.8.1-src]#

编译的时间会比较久

8.编译好的tar包

 /opt/sourcecode/hadoop-2.8.1-src/hadoop-dist/target/hadoop-2.8.1.tar.gz

提醒:

1、有时候编译过程中会出现下载某个包的时间太久,这是由于连接网站的过程中会出现假死,

此时按ctrl+c,重新运行编译命令。

2、如果出现缺少了某个文件的情况,则要先清理maven(使用命令 mvn clean) 再重新编译。

© 著作权归作者所有

共有 人打赏支持
GordonNemo
粉丝 4
博文 30
码字总数 52159
作品 0
杭州
架构师
编译hadoop-2.2.0源码,RHEL6.3x64

编译环境: OS: RHEL6.3.x86_64 Kernel:2.6.32-358.el6.x86_64 JDK: 1.6.035 x8664 编译前准备: 1.安装依赖的rpm包 yum -y install lzo-devel zlib-devel gcc autoconf automake libtool op......

徐霞客 ⋅ 2013/12/28 ⋅ 0

重新编译hadoop源代码

hadoop-1.0.3源代码下载地址: http://svn.apache.org/repos/asf/hadoop/common/tags/release-1.0.3/ 实验环境: redhat版本 Red Hat Enterprise Linux Server release 6.5 (Santiago) 在red......

hiqj ⋅ 2015/05/18 ⋅ 0

hadoop2.7环境的编译安装

0.环境说明 编译所用的操作系统为: [root@host11 hadoop-2.7.1-src]# cat /etc/redhat-release CentOS release 6.5 (Final) hadoop的版本为2.7.1 1.安装依赖软件包 yum install svn autoconf......

openthings ⋅ 2015/12/17 ⋅ 2

如何在 Hadoop 2.2.0 环境下使用 Spark 2.2.x

如果下面文章格式或图片不清晰,请参见原文:https://www.iteblog.com/archives/2305.html 或直接点击下面 阅读原文 即可进入。 到 Github 下载支持 Hadoop 2.5 及其之前版本的 Spark 2.2.1 ...

b6ecl1k7bs8o ⋅ 2017/12/14 ⋅ 0

YARN加载本地库Unable to load native-hadoop library解决办法

用官方的构建好的Hadoop 2.4.0(133MB)安装后,每次输入hadoop命令进去都会得到这样一个Warning,如图: WARN util.NativeCodeLoader: Unable to load native-hadoop library for your plat...

摆渡者 ⋅ 2014/05/28 ⋅ 0

linux centos 安装编译hadoop2.7.1

因为Apache hadoop 只提供了32位的包,所以需要自己编译64位的包,下面是编译步骤 所需要的包 1:apache-ant-1.9.7-bin.tar.gz(http://mirrors.tuna.tsinghua.edu.cn/apache//ant/binaries/...

不正常的物种 ⋅ 2016/09/12 ⋅ 0

hadoop开发环境搭建

hadoop开发环境搭建: hadoop网站: http://hadoop.apache.org/ http://wiki.apache.org/hadoop/ 参考这篇文章编译:http://wiki.apache.org/hadoop/HowToContribute 开发环境:ubuntu 需要安......

活的自由 ⋅ 2011/10/19 ⋅ 1

如何在 Hadoop 2.2.0 环境下使用 Spark 2.2.x

Apache Spark 2.2.0 于今年7月份正式发布, 这个版本是 Structured Streaming 的一个重要里程碑,因为其可以正式在生产环境中使用,实验标签(experimental tag)已经被移除; CBO (Cost-Ba...

Spark ⋅ 2017/12/13 ⋅ 0

Nutch问题求高手指导

我是一个Nutch生手,最近在搭建Nutch,已经成功安装好java、cygwin(类似linux虚拟机的一个东东)、tomcat、Nutch-1.9(发布版),并配置好环境并测试成功,但在输入抓取数据指令(注:urls.txt里保...

qubo3000 ⋅ 2015/02/02 ⋅ 0

TEZ计算引擎部署

一、tez简介 tez官网:http://tez.apache.org 在使用tez作为计算引擎使用之前先说明下tez-ui。tez-ui是查看tez任务执行日志的的web界面,依赖于yarn的timeline服务。tez0.8.3中又增加了tez-u...

Yulong_ ⋅ 2016/04/25 ⋅ 0

没有更多内容

加载失败,请刷新页面

加载更多

下一页

Springboot2 之 Spring Data Redis 实现消息队列——发布/订阅模式

一般来说,消息队列有两种场景,一种是发布者订阅者模式,一种是生产者消费者模式,这里利用redis消息“发布/订阅”来简单实现订阅者模式。 实现之前先过过 redis 发布订阅的一些基础概念和操...

Simonton ⋅ 27分钟前 ⋅ 0

error:Could not find gradle

一.更新Android Studio后打开Project,报如下错误: Error: Could not find com.android.tools.build:gradle:2.2.1. Searched in the following locations: file:/D:/software/android/andro......

Yao--靠自己 ⋅ 昨天 ⋅ 0

Spring boot 项目打包及引入本地jar包

Spring Boot 项目打包以及引入本地Jar包 [TOC] 上篇文章提到 Maven 项目添加本地jar包的三种方式 ,本篇文章记录下在实际项目中的应用。 spring boot 打包方式 我们知道,传统应用可以将程序...

Os_yxguang ⋅ 昨天 ⋅ 0

常见数据结构(二)-树(二叉树,红黑树,B树)

本文介绍数据结构中几种常见的树:二分查找树,2-3树,红黑树,B树 写在前面 本文所有图片均截图自coursera上普林斯顿的课程《Algorithms, Part I》中的Slides 相关命题的证明可参考《算法(第...

浮躁的码农 ⋅ 昨天 ⋅ 0

android -------- 混淆打包报错 (warning - InnerClass ...)

最近做Android混淆打包遇到一些问题,Android Sdutio 3.1 版本打包的 错误如下: Android studio warning - InnerClass annotations are missing corresponding EnclosingMember annotation......

切切歆语 ⋅ 昨天 ⋅ 0

eclipse酷炫大法之设置主题、皮肤

eclipse酷炫大法 目前两款不错的eclipse 1.系统设置 Window->Preferences->General->Appearance 2.Eclipse Marketplace下载【推荐】 Help->Eclipse Marketplace->搜索‘theme’进行安装 比如......

anlve ⋅ 昨天 ⋅ 0

vim编辑模式、vim命令模式、vim实践

vim编辑模式 编辑模式用来输入或修改文本内容,编辑模式除了Esc外其他键几乎都是输入 如何进入编辑模式 一般模式输入以下按键,均可进入编辑模式,左下角提示 insert(中文为插入) 字样 i ...

蛋黄Yolks ⋅ 昨天 ⋅ 0

大数据入门基础:SSH介绍

什么是ssh 简单说,SSH是一种网络协议,用于计算机之间的加密登录。 如果一个用户从本地计算机,使用SSH协议登录另一台远程计算机,我们就可以认为,这种登录是安全的,即使被中途截获,密码...

董黎明 ⋅ 昨天 ⋅ 0

web3j教程

web3j是一个轻量级、高度模块化、响应式、类型安全的Java和Android类库提供丰富API,用于处理以太坊智能合约及与以太坊网络上的客户端(节点)进行集成。 汇智网最新发布的web3j教程,详细讲解...

汇智网教程 ⋅ 昨天 ⋅ 0

谷歌:安全问题机制并不如你想象中安全

腾讯科技讯 5月25日,如今的你或许已经对许多网站所使用的“安全问题机制”习以为常了,但你真的认为包括“你第一个宠物的名字是什么?”这些问题能够保障你的帐户安全吗? 根据谷歌(微博)安...

问题终结者 ⋅ 昨天 ⋅ 0

没有更多内容

加载失败,请刷新页面

加载更多

下一页

返回顶部
顶部