start-all 启动分析
set HADOOP_BIN_PATH = ${HADOOP_INSTALL}\sbin
set HADOOP_LIBEXEC_DIR = ${HADOOP_INSTALL}\libexec
${HADOOP_INSTALL}\libexec\hadoop-config.cmd #设置环境变量
%HADOOP_CONF_DIR%\hadoop-env.cmd
set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_DIR%\*
#设置hadoop 可执行路径
#设置 LIBEXEC 可执行类库目录
# 调用 libexec\hdfs-config.cmd 命令
${HADOOP_INSTALL}\sbin\start-dfs.cmd --config %HADOOP_CONF_DIR%
${HADOOP_INSTALL}\libexec\hdfs-config.cmd
start "Apache Hadoop Distribution" hadoop.cmd namenode
call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %*
call %HADOOP_HOME%\bin\hdfs.cmd %*
call %HADOOP_LIBEXEC_DIR%\hdfs-config.cmd %*
#参考上面的分析,主要设置环境变量
call %HADOOP_CONF_DIR%\hadoop-env.cmd
#设置JAVA_HOME,设置Namenode,datanode,secondNamenode 节点及其选项
set java_arguments=%JAVA_HEAP_MAX% %HADOOP_OPTS%
-classpath %CLASSPATH% %CLASS% %hdfs-command-arguments%
call %JAVA% %java_arguments%
#设置mapredce命令组
set mapredcommands=pipes job queue mrgroups mradmin jobtracker tasktracker
call %HADOOP_HOME%\bin\mapred.cmd %*
call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %*
#参考上面的分析,主要设置环境变量
call %HADOOP_CONF_DIR%\mapred-env.cmd
#
set HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
set HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
//classpath 既不是hdfs命令,也不是mapreduce命令
Command[CLASSPATH] //处理classpath命令
//设置核心命令组,
set corecommands=fs version jar checknative distcp daemonlog archive
//处理核心命令,将对应名称解析成对应class完整名
call :%hadoop-command% //调用相应的命名标签
//调用JAVA程序
call %JAVA% %JAVA_HEAP_MAX% %HADOOP_OPTS%
-classpath %CLASSPATH% %CLASS% %hadoop-command-arguments%
start "Apache Hadoop Distribution" hadoop.cmd datanode
#参考 hadoop.cmd datanode 指令
${HADOOP_INSTALL}\sbin\start-yarn.cmd --config %HADOOP_CONF_DIR%
set HADOOP_BIN_PATH=%~dp0
call %HADOOP_LIBEXEC_DIR%\yarn-config.cmd %*
#参考上面的分析,主要设置环境变量
call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %*
#参考上面的分析,主要设置环境变量
start "Apache Hadoop Distribution" yarn resourcemanager
call %HADOOP_LIBEXEC_DIR%\yarn-config.cmd %*
#参考上面的分析,主要设置环境变量
call %YARN_CONF_DIR%\yarn-env.cmd
call :make_command_arguments %*
call :%yarn-command% %yarn-command-arguments%
set YARN_OPTS=%YARN_OPTS% -Djava.library.path=%JAVA_LIBRARY_PATH%
set java_arguments=%JAVA_HEAP_MAX% %YARN_OPTS%
-classpath %CLASSPATH% %CLASS% %yarn-command-arguments%
call %JAVA% %java_arguments%
start "Apache Hadoop Distribution" yarn nodemanager
start "Apache Hadoop Distribution" yarn proxyserver
start-all 启动分析
set HADOOP_BIN_PATH = ${HADOOP_INSTALL}\sbin
set HADOOP_LIBEXEC_DIR = ${HADOOP_INSTALL}\libexec
${HADOOP_INSTALL}\libexec\hadoop-config.cmd #设置环境变量
set HADOOP_COMMON_DIR=share\hadoop\common
set HADOOP_COMMON_LIB_JARS_DIR=share\hadoop\common\lib
set HADOOP_COMMON_LIB_NATIVE_DIR=lib\native
set HDFS_DIR=share\hadoop\hdfs
set HDFS_LIB_JARS_DIR=share\hadoop\hdfs\lib
set YARN_DIR=share\hadoop\yarn
set YARN_LIB_JARS_DIR=share\hadoop\yarn\lib
set MAPRED_DIR=share\hadoop\mapreduce
set MAPRED_LIB_JARS_DIR=share\hadoop\mapreduce\lib
单独判断 %HADOOP_HOME%\share\hadoop\common\hadoop-common-*.jar 是否存在
set HADOOP_CONF_DIR=%HADOOP_HOME%\etc\hadoop
%HADOOP_CONF_DIR%\hadoop-env.cmd
#设置JAVA_HOME,设置Namenode,datanode,secondNamenode 节点及其选项
set JAVA_HOME=%JAVA_HOME%
set HADOOP_CLASSPATH=%HADOOP_HOME%\contrib\capacity-scheduler\*.jar
set HADOOP_NAMENODE_OPTS=-Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER%
-Dhdfs.audit.logger=%HDFS_AUDIT_LOGGER% %HADOOP_NAMENODE_OPTS%
set HADOOP_DATANODE_OPTS=-Dhadoop.security.logger=ERROR,RFAS %HADOOP_DATANODE_OPTS%
set HADOOP_SECONDARYNAMENODE_OPTS=-Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER%
-Dhdfs.audit.logger=%HDFS_AUDIT_LOGGER% %HADOOP_SECONDARYNAMENODE_OPTS%
set HADOOP_CLIENT_OPTS=-Xmx128m %HADOOP_CLIENT_OPTS%
set JAVA=%JAVA_HOME%\bin\java
set JAVA_HEAP_MAX=-Xmx1000m //设置虚拟机参数:最大堆内存
set JAVA_HEAP_MAX=-Xmx%HADOOP_HEAPSIZE%m hadoop对覆盖java中的-Xmx 参数
set CLASSPATH=%HADOOP_CONF_DIR%
set CLASSPATH=!CLASSPATH!;%HADOOP_COMMON_HOME%\%HADOOP_COMMON_DIR%
set CLASSPATH=!CLASSPATH!;%HADOOP_COMMON_HOME%\%HADOOP_COMMON_LIB_JARS_DIR%\*
set CLASSPATH=!CLASSPATH!;%HADOOP_COMMON_HOME%\%HADOOP_COMMON_DIR%\*
set CLASSPATH=%CLASSPATH%;%HADOOP_CLASSPATH%;
set HADOOP_HDFS_HOME=%HADOOP_HOME%
set CLASSPATH=!CLASSPATH!;%HADOOP_HDFS_HOME%\%HDFS_DIR%
set CLASSPATH=!CLASSPATH!;%HADOOP_HDFS_HOME%\%HDFS_LIB_JARS_DIR%\*
set CLASSPATH=!CLASSPATH!;%HADOOP_HDFS_HOME%\%HDFS_DIR%\*
set HADOOP_YARN_HOME=%HADOOP_HOME%
set CLASSPATH=!CLASSPATH!;%HADOOP_YARN_HOME%\%YARN_DIR%
set CLASSPATH=!CLASSPATH!;%HADOOP_YARN_HOME%\%YARN_LIB_JARS_DIR%\*
set CLASSPATH=!CLASSPATH!;%HADOOP_YARN_HOME%\%YARN_DIR%\*
set HADOOP_MAPRED_HOME=%HADOOP_HOME%
set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_DIR%
set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_LIB_JARS_DIR%\*
set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_DIR%\*
#设置hadoop 可执行路径
#设置 LIBEXEC 可执行类库目录
# 调用 libexec\hdfs-config.cmd 命令
${HADOOP_INSTALL}\sbin\start-dfs.cmd --config %HADOOP_CONF_DIR%
set HADOOP_BIN_PATH=${HADOOP_INSTALL}\sbin
${HADOOP_INSTALL}\libexec\hdfs-config.cmd
set HADOOP_BIN_PATH=%~dp0
set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %*
...参考上面的分析,主要设置环境变量
判断是否定义 --config 参数
#start:启动单独的窗口运行新命令
start "Apache Hadoop Distribution" hadoop.cmd namenode
set HADOOP_BIN_PATH=%~dp0
#:updatepath 为之前定义的标签
call :updatepath %HADOOP_BIN_PATH% #更新路径
set path_to_add=%*
set current_path_comparable=%path%
#用下划线替换替换变量中的空格
set current_path_comparable=%current_path_comparable: =_%
#用下划线替换替换变量中的(
set current_path_comparable=%current_path_comparable:(=_%
set current_path_comparable=%current_path_comparable:)=_%
set path_to_add_comparable=%path_to_add%
set path_to_add_comparable=%path_to_add_comparable: =_%
set path_to_add_comparable=%path_to_add_comparable:(=_%
set path_to_add_comparable=%path_to_add_comparable:)=_%
......参考上面的分析,主要设置环境变量
set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR%
call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %*
set hadoop-command=%1 #提取第一个参数
call :make_command_arguments %* #处理命令参数
set hadoop-command-arguments=%_arguments% #设置hadoop-command环境变量
#设置hdfs命令组
set hdfscommands=namenode secondarynamenode datanode dfs
dfsadmin fsck balancer fetchdt oiv dfsgroups
call %HADOOP_HOME%\bin\hdfs.cmd %*
set HADOOP_BIN_PATH=%~dp0
set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
call %HADOOP_LIBEXEC_DIR%\hdfs-config.cmd %*
#参考上面的分析,主要设置环境变量
call %HADOOP_CONF_DIR%\hadoop-env.cmd
#设置JAVA_HOME,设置Namenode,datanode,secondNamenode 节点及其选项
set hdfs-command=%1
call :make_command_arguments %*
set hdfs-command-arguments=%_hdfsarguments%
如果命令无效,goto print_usage
call :%hdfs-command% %hdfs-command-arguments%
:namenode
set CLASS=org.apache.hadoop.hdfs.server.namenode.NameNode
set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_NAMENODE_OPTS%
:zkfc
set CLASS=org.apache.hadoop.hdfs.tools.DFSZKFailoverController
set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_ZKFC_OPTS%
:secondarynamenode
set CLASS=org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode
set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_SECONDARYNAMENODE_OPTS%
:datanode
set CLASS=org.apache.hadoop.hdfs.server.datanode.DataNode
set HADOOP_OPTS=%HADOOP_OPTS% -server %HADOOP_DATANODE_OPTS%
:dfs
set CLASS=org.apache.hadoop.fs.FsShell
set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS%
:dfsadmin
set CLASS=org.apache.hadoop.hdfs.tools.DFSAdmin
set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS%
:haadmin
set CLASS=org.apache.hadoop.hdfs.tools.DFSHAAdmin
set CLASSPATH=%CLASSPATH%;%TOOL_PATH%
set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS%
:fsck
set CLASS=org.apache.hadoop.hdfs.tools.DFSck
set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS%
:balancer
set CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer
set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_BALANCER_OPTS%
:jmxget
set CLASS=org.apache.hadoop.hdfs.tools.JMXGet
:oiv
set CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer
:oev
set CLASS=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer
:fetchdt
set CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
:getconf
set CLASS=org.apache.hadoop.hdfs.tools.GetConf
:groups
set CLASS=org.apache.hadoop.hdfs.tools.GetGroups
set java_arguments=%JAVA_HEAP_MAX% %HADOOP_OPTS%
-classpath %CLASSPATH% %CLASS% %hdfs-command-arguments%
call %JAVA% %java_arguments%
#设置mapredce命令组
set mapredcommands=pipes job queue mrgroups mradmin jobtracker tasktracker
call %HADOOP_HOME%\bin\mapred.cmd %*
set HADOOP_BIN_PATH=%~dp0
set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %*
#参考上面的分析,主要设置环境变量
call %HADOOP_CONF_DIR%\mapred-env.cmd
#
set HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
set HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
//classpath 既不是hdfs命令,也不是mapreduce命令
Command[CLASSPATH] //处理classpath命令
goto :eof
//设置核心命令组,
set corecommands=fs version jar checknative distcp daemonlog archive
//处理核心命令,将对应名称解析成对应class完整名
call :%hadoop-command% //调用相应的命名标签
:fs set CLASS=org.apache.hadoop.fs.FsShell
:version set CLASS=org.apache.hadoop.util.VersionInfo
:jar set CLASS=org.apache.hadoop.util.RunJar
:checknative set CLASS=org.apache.hadoop.tools.DistCp
:distcp set CLASS=org.apache.hadoop.fs.FsShell
:daemonlog set CLASS=org.apache.hadoop.log.LogLevel
:archive set CLASS=org.apache.hadoop.tools.HadoopArchives
set CLASSPATH=%CLASSPATH%;%TOOL_PATH%
//把命令名称作为CLASS值.
set CLASSPATH=%CLASSPATH%;%CD%
set CLASS=%hadoop-command%
set path=%PATH%;%HADOOP_BIN_PATH%
set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS%
set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER%
//调用JAVA程序
call %JAVA% %JAVA_HEAP_MAX% %HADOOP_OPTS%
-classpath %CLASSPATH% %CLASS% %hadoop-command-arguments%
:goto eof
start "Apache Hadoop Distribution" hadoop.cmd datanode
#参考 hadoop.cmd datanode 指令
${HADOOP_INSTALL}\sbin\start-yarn.cmd --config %HADOOP_CONF_DIR%
set HADOOP_BIN_PATH=%~dp0
set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
call %HADOOP_LIBEXEC_DIR%\yarn-config.cmd %*
#参考上面的分析,主要设置环境变量
set HADOOP_BIN_PATH=%~dp0
set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %*
#参考上面的分析,主要设置环境变量
set YARN_CONF_DIR=%2
set YARN_CONF_DIR=%HADOOP_YARN_HOME%\conf
set YARN_SLAVES=%YARN_CONF_DIR%\%2
start "Apache Hadoop Distribution" yarn resourcemanager
set HADOOP_BIN_PATH=%~dp0
set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
call %HADOOP_LIBEXEC_DIR%\yarn-config.cmd %*
#参考上面的分析,主要设置环境变量
call %YARN_CONF_DIR%\yarn-env.cmd
set HADOOP_YARN_USER=%yarn%
set JAVA_HEAP_MAX=-Xmx%YARN_HEAPSIZE%m
set YARN_LOG_DIR=%HADOOP_YARN_HOME%\logs
set YARN_LOGFILE=yarn.log
set YARN_POLICYFILE=hadoop-policy.xml
set YARN_ROOT_LOGGER=INFO,console
set YARN_OPTS=%YARN_OPTS% -Dhadoop.log.dir=%YARN_LOG_DIR%
set YARN_OPTS=%YARN_OPTS% -Dyarn.log.dir=%YARN_LOG_DIR%
set YARN_OPTS=%YARN_OPTS% -Dhadoop.log.file=%YARN_LOGFILE%
set YARN_OPTS=%YARN_OPTS% -Dyarn.log.file=%YARN_LOGFILE%
set YARN_OPTS=%YARN_OPTS% -Dyarn.home.dir=%HADOOP_YARN_HOME%
set YARN_OPTS=%YARN_OPTS% -Dyarn.id.str=%YARN_IDENT_STRING%
set YARN_OPTS=%YARN_OPTS% -Dhadoop.home.dir=%HADOOP_YARN_HOME%
set YARN_OPTS=%YARN_OPTS% -Dhadoop.root.logger=%YARN_ROOT_LOGGER%
set YARN_OPTS=%YARN_OPTS% -Dyarn.root.logger=%YARN_ROOT_LOGGER%
set YARN_OPTS=%YARN_OPTS% -Djava.library.path=%JAVA_LIBRARY_PATH%
set YARN_OPTS=%YARN_OPTS% -Dyarn.policy.file=%YARN_POLICYFILE%
#将第一个参数设置为yarn命令
set yarn-command=%1
call :make_command_arguments %*
set yarn-command-arguments=%_yarnarguments%
set JAVA_HEAP_MAX=-Xmx%YARN_HEAPSIZE%m
set CLASSPATH=%HADOOP_CONF_DIR%;%YARN_CONF_DIR%;%CLASSPATH%
set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\share\hadoop\yarn\*
set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_LIB_JARS_DIR%\*
call :%yarn-command% %yarn-command-arguments%
:classpath
@echo %CLASSPATH%
:rmadmin
set CLASS=org.apache.hadoop.yarn.server.resourcemanager.tools.RMAdmin
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
:application
set CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
:node
set CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
:resourcemanager
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\rm-config\log4j.properties
set CLASS=org.apache.hadoop.yarn.server.resourcemanager.ResourceManager
set YARN_OPTS=%YARN_OPTS% %HADOOP_RESOURCEMANAGER_OPTS%
if defined YARN_RESOURCEMANAGER_HEAPSIZE (
set JAVA_HEAP_MAX=-Xmx%YARN_RESOURCEMANAGER_HEAPSIZE%m
)
:nodemanager
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\nm-config\log4j.properties
set CLASS=org.apache.hadoop.yarn.server.nodemanager.NodeManager
set YARN_OPTS=%YARN_OPTS% -server %HADOOP_NODEMANAGER_OPTS%
if defined YARN_NODEMANAGER_HEAPSIZE (
set JAVA_HEAP_MAX=-Xmx%YARN_NODEMANAGER_HEAPSIZE%m
)
:proxyserver
set CLASS=org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer
set YARN_OPTS=%YARN_OPTS% %HADOOP_PROXYSERVER_OPTS%
if defined YARN_PROXYSERVER_HEAPSIZE (
set JAVA_HEAP_MAX=-Xmx%YARN_PROXYSERVER_HEAPSIZE%m
)
:version
set CLASS=org.apache.hadoop.util.VersionInfo
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
:jar
set CLASS=org.apache.hadoop.util.RunJar
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
:logs
set CLASS=org.apache.hadoop.yarn.logaggregation.LogDumper
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
:daemonlog
set CLASS=org.apache.hadoop.log.LogLevel
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
set YARN_OPTS=%YARN_OPTS% -Djava.library.path=%JAVA_LIBRARY_PATH%
set java_arguments=%JAVA_HEAP_MAX% %YARN_OPTS%
-classpath %CLASSPATH% %CLASS% %yarn-command-arguments%
call %JAVA% %java_arguments%
start "Apache Hadoop Distribution" yarn nodemanager
start "Apache Hadoop Distribution" yarn proxyserver