{"id":109,"date":"2019-12-30T18:59:59","date_gmt":"2019-12-30T10:59:59","guid":{"rendered":"http:\/\/blog.73007300.xyz\/?p=109"},"modified":"2019-12-30T18:59:59","modified_gmt":"2019-12-30T10:59:59","slug":"hadoop-yarn-%e6%9c%aa%e9%89%b4%e6%9d%83%e6%89%a7%e8%a1%8c%e5%91%bd%e4%bb%a4","status":"publish","type":"post","link":"https:\/\/blog.73007300.xyz\/?p=109","title":{"rendered":"Hadoop Yarn \u672a\u9274\u6743\u6267\u884c\u547d\u4ee4"},"content":{"rendered":"\n<h2 class=\"wp-block-heading\">\u5b89\u88c5<\/h2>\n\n\n\n<p>\u6211\u7528\u7684\u662f hadoop-3.1.3 \u4e0b\u8f7d\uff0c\u89e3\u538b\u3002\u914d\u7f6eJDK\u73af\u5883\u53d8\u91cf \u548c \u5176\u4ed6\u73af\u5883\u53d8\u91cf\u3002<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>export JAVA_HOME=\/home\/hans\/JDK\/jdk1.8.0_192\nexport PATH=$JAVA_HOME\/bin:$JAVA_HOME\/jre\/bin:$PATH\nexport HDFS_NAMENODE_USER=\"root\"\nexport HDFS_DATANODE_USER=\"root\"\nexport HDFS_SECONDARYNAMENODE_USER=\"root\"\nexport YARN_RESOURCEMANAGER_USER=\"root\"\nexport YARN_NODEMANAGER_USER=\"root\"\nexport PDSH_RCMD_TYPE=ssh<\/code><\/pre>\n\n\n\n<h2 class=\"wp-block-heading\">\u914d\u7f6e<\/h2>\n\n\n\n<p>\u4fee\u6539\u56db\u4e2a\u6587\u4ef6\uff1a<br>\nhadoop-3.1.3\/etc\/hadoop\/core-site.xml<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>&lt;?xml version=\"1.0\" encoding=\"UTF-8\"?>\n&lt;?xml-stylesheet type=\"text\/xsl\" href=\"configuration.xsl\"?>\n&lt;configuration>\n    &lt;property>\n        &lt;name>fs.defaultFS&lt;\/name>\n        &lt;value>hdfs:\/\/localhost:9000&lt;\/value>\n    &lt;\/property>\n&lt;\/configuration><\/code><\/pre>\n\n\n\n<p>hadoop-3.1.3\/etc\/hadoop\/\/hdfs-site.xml<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>&lt;?xml version=\"1.0\" encoding=\"UTF-8\"?>\n&lt;?xml-stylesheet type=\"text\/xsl\" href=\"configuration.xsl\"?>\n&lt;configuration>\n    &lt;property>\n        &lt;name>dfs.replication&lt;\/name>\n        &lt;value>1&lt;\/value>\n    &lt;\/property>\n&lt;\/configuration><\/code><\/pre>\n\n\n\n<p>hadoop-3.1.3\/etc\/hadoop\/mapred-site.xml<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>&lt;?xml version=\"1.0\"?>\n&lt;?xml-stylesheet type=\"text\/xsl\" href=\"configuration.xsl\"?>\n&lt;configuration> \n    &lt;property>\n        &lt;name>mapreduce.framework.name&lt;\/name>\n        &lt;value>yarn&lt;\/value>\n    &lt;\/property>\n    &lt;property>\n        &lt;name>mapreduce.application.classpath&lt;\/name>\n        &lt;value>$HADOOP_MAPRED_HOME\/share\/hadoop\/mapreduce\/*:$HADOOP_MAPRED_HOME\/share\/hadoop\/mapreduce\/lib\/*&lt;\/value>\n    &lt;\/property>\n&lt;\/configuration><\/code><\/pre>\n\n\n\n<p>hadoop-3.1.3\/etc\/hadoop\/yarn-site.xml<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>&lt;?xml version=\"1.0\"?>\n&lt;configuration>\n    &lt;property>\n        &lt;name>yarn.nodemanager.aux-services&lt;\/name>\n        &lt;value>mapreduce_shuffle&lt;\/value>\n    &lt;\/property>\n    &lt;property>\n        &lt;name>yarn.nodemanager.env-whitelist&lt;\/name>\n        &lt;value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME&lt;\/value>\n    &lt;\/property>\n&lt;\/configuration><\/code><\/pre>\n\n\n\n<h2 class=\"wp-block-heading\">\u542f\u52a8<\/h2>\n\n\n\n<pre class=\"wp-block-code\"><code>bin\/hdfs namenode -format\nsbin\/start-yarn.sh<\/code><\/pre>\n\n\n\n<h2 class=\"wp-block-heading\">\u5229\u7528\uff1a<\/h2>\n\n\n\n<p>\u521b\u5efa\u65b0\u7684application<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>curl -v -X POST 'http:\/\/192.168.23.134:8088\/ws\/v1\/cluster\/apps\/new-application'<\/code><\/pre>\n\n\n\n<p>\u521b\u5efa\u5e76\u4fee\u6539json\u6587\u4ef6\u3002<br>\n1.json\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>{  \n    \"am-container-spec\":{  \n        \"commands\":{  \n            \"command\":\"echo '111' > \/tmp\/11112222_test_11112222\"\n\n        }  \n    },  \n    \"application-id\":\"application_1576573490143_0003\",  \n    \"application-name\":\"test\",  \n    \"application-type\":\"YARN\"  \n} <\/code><\/pre>\n\n\n\n<p>application-id \u4fee\u6539\u4e3a \u521b\u5efa\u65b0\u7684application \u8fd4\u56de\u7684ID\u3002<\/p>\n\n\n\n<p>\u53d1\u9001\u5229\u7528\u547d\u4ee4\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>curl -s -i -X POST -H 'Accept: application\/json' -H 'Content-Type: application\/json' 'http:\/\/192.168.23.134:8088\/ws\/v1\/cluster\/apps' --data-binary @1.json<\/code><\/pre>\n\n\n\n<p><a href=\"http:\/\/hadoop.apache.org\/docs\/current\/hadoop-yarn\/hadoop-yarn-site\/ResourceManagerRest.html#Cluster_Applications_API\">http:\/\/hadoop.apache.org\/docs\/current\/hadoop-yarn\/hadoop-yarn-site\/ResourceManagerRest.html#Cluster_Applications_API<\/a><\/p>\n\n\n\n<p><a href=\"https:\/\/hadoop.apache.org\/docs\/r3.1.1\/hadoop-project-dist\/hadoop-common\/HttpAuthentication.html\">https:\/\/hadoop.apache.org\/docs\/r3.1.1\/hadoop-project-dist\/hadoop-common\/HttpAuthentication.html<\/a><\/p>\n\n\n\n<p><a href=\"https:\/\/hadoop.apache.org\/docs\/r3.1.1\/hadoop-project-dist\/hadoop-common\/SingleCluster.html\">https:\/\/hadoop.apache.org\/docs\/r3.1.1\/hadoop-project-dist\/hadoop-common\/SingleCluster.html<\/a><\/p>\n\n\n\n<p><a href=\"https:\/\/paper.seebug.org\/611\/\">https:\/\/paper.seebug.org\/611\/<\/a><\/p>\n\n\n\n<h2 class=\"wp-block-heading\">\u4fee\u590d\u65b9\u6848<\/h2>\n\n\n\n<ol><li>\u66f4\u65b0Hadoop\u5230\u6700\u65b0\u7248\u672c\u5e76\u542f\u7528Kerberos\u8ba4\u8bc1\u529f\u80fd\uff0c\u7981\u6b62\u533f\u540d\u8bbf\u95ee\uff1b<\/li><li> \u914d\u7f6eiptables\u6216\u5b89\u5168\u7ec4\u7b56\u7565\u5b9e\u65bd\u8bbf\u95ee\u63a7\u5236\uff0c\u7981\u6b62\u4e0d\u53ef\u4fe1IP\u8fdb\u884c\u8bbf\u95ee\uff1b\u82e5\u65e0\u5fc5\u8981\uff0c\u7aef\u53e3\u4e0d\u8981\u76d1\u542c\u5728\u516c\u7f51\uff0c\u6539\u4e3a\u76d1\u542c\u672c\u5730\u5730\u5740\u6216\u8005\u5185\u7f51\u5730\u5740\u3002<\/li><\/ol>\n","protected":false},"excerpt":{"rendered":"<p>\u5b89\u88c5 \u6211\u7528\u7684\u662f hadoop-3.1.3 \u4e0b\u8f7d\uff0c\u89e3\u538b\u3002\u914d\u7f6eJDK\u73af\u5883\u53d8\u91cf \u548c \u5176\u4ed6\u73af\u5883\u53d8\u91cf\u3002 \u914d\u7f6e \u4fee\u6539\u56db\u4e2a\u6587\u4ef6\uff1a hadoop-3.1.3\/etc\/hadoop\/core-site.xml hadoop-3.1.3\/etc\/hadoop\/\/hdfs-site.xml hadoop-3.1.3\/etc\/hadoop\/mapred-site.xml hadoop-3.1.3\/etc\/hadoop\/yarn-site.xml \u542f\u52a8 \u5229\u7528\uff1a \u521b\u5efa\u65b0\u7684application \u521b\u5efa\u5e76\u4fee\u6539json\u6587\u4ef6\u3002 1.json\uff1a application-id \u4fee\u6539\u4e3a \u521b\u5efa\u65b0\u7684application \u8fd4\u56de\u7684ID\u3002 \u53d1\u9001\u5229\u7528\u547d\u4ee4\uff1a http:\/\/hadoop.apache.org\/docs\/current\/hadoop-yarn\/hadoop-yarn-site\/ResourceManagerRest.html#Cluster_Applications_API https:\/\/hadoop.apache.org\/docs\/r3.1.1\/hadoop-project-dist\/hadoop-common\/HttpAuthentication.html https:\/\/hadoop.apache.org\/docs\/r3.1.1\/hadoop-project-dist\/hadoop-common\/SingleCluster.html https:\/\/paper.seebug.org\/611\/ \u4fee\u590d\u65b9\u6848 \u66f4\u65b0Hadoop\u5230\u6700\u65b0\u7248\u672c\u5e76\u542f\u7528Kerberos\u8ba4\u8bc1\u529f\u80fd\uff0c\u7981\u6b62\u533f\u540d\u8bbf\u95ee\uff1b \u914d\u7f6eiptables\u6216\u5b89\u5168\u7ec4\u7b56\u7565\u5b9e\u65bd\u8bbf\u95ee\u63a7\u5236\uff0c\u7981\u6b62\u4e0d\u53ef\u4fe1IP\u8fdb\u884c\u8bbf\u95ee\uff1b\u82e5\u65e0\u5fc5\u8981\uff0c\u7aef\u53e3\u4e0d\u8981\u76d1\u542c\u5728\u516c\u7f51\uff0c\u6539\u4e3a\u76d1\u542c\u672c\u5730\u5730\u5740\u6216\u8005\u5185\u7f51\u5730\u5740\u3002<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[4],"tags":[],"_links":{"self":[{"href":"https:\/\/blog.73007300.xyz\/index.php?rest_route=\/wp\/v2\/posts\/109"}],"collection":[{"href":"https:\/\/blog.73007300.xyz\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/blog.73007300.xyz\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/blog.73007300.xyz\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/blog.73007300.xyz\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=109"}],"version-history":[{"count":1,"href":"https:\/\/blog.73007300.xyz\/index.php?rest_route=\/wp\/v2\/posts\/109\/revisions"}],"predecessor-version":[{"id":110,"href":"https:\/\/blog.73007300.xyz\/index.php?rest_route=\/wp\/v2\/posts\/109\/revisions\/110"}],"wp:attachment":[{"href":"https:\/\/blog.73007300.xyz\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=109"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/blog.73007300.xyz\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=109"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/blog.73007300.xyz\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=109"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}