hadoop内存配置方案

Configuration File

Configuration Setting

Value Calculation

       8G VM (4G For MR)  

yarn-site.xml

yarn.nodemanager.resource.memory-mb

= containers * RAM-per-container

4096

yarn-site.xml

yarn.scheduler.minimum-allocation-mb

= RAM-per-container

1024

yarn-site.xml

yarn.scheduler.maximum-allocation-mb

= containers * RAM-per-container

4096

mapred-site.xml

mapreduce.map.memory.mb

= RAM-per-container

1024

mapred-site.xml       

mapreduce.reduce.memory.mb

= 2 * RAM-per-container

2048

mapred-site.xml

mapreduce.map.java.opts

= 0.8 * RAM-per-container

819

mapred-site.xml

mapreduce.reduce.java.opts

= 0.8 * 2 * RAM-per-container

1638

yarn-site.xml (check)

yarn.app.mapreduce.am.resource.mb

= 2 * RAM-per-container

2048

yarn-site.xml (check)

yarn.app.mapreduce.am.command-opts

= 0.8 * 2 * RAM-per-container

1638

tez-site.xml 

tez.am.resource.memory.mb 

= RAM-per-container

1024

tez-site.xml 

tez.am.java.opts 

= 0.8 * RAM-per-container

819

tez-site.xml 

hive.tez.container.size 

= RAM-per-container

1024

tez-site.xml 

hive.tez.java.opts 

= 0.8 * RAM-per-container

819