ansible-role-hadoop-datanode

Install datanode of hadoop.

Requirements

None

Role Variables

variable	description	default
hadoop_datanode_user	datanode user	"{{ __hadoop_datanode_user }}"
hadoop_datanode_group	datanode group	"{{ __hadoop_datanode_group }}"
hadoop_datanode_log_dir	path to log dir	/var/log/hadoop
hadoop_datanode_db_dir	path to db dir	"{{ __hadoop_datanode_db_dir }}"
hadoop_datanode_service	service name	"{{ __hadoop_datanode_service }}"
hadoop_datanode_nodemanager_service	nodemanager service name	"{{ __hadoop_datanode_nodemanager_service }}"
hadoop_datanode_conf_dir	path to config dir	"{{ __hadoop_datanode_conf_dir }}"
hadoop_datanode_flags	not used	""
hadoop_datanode_dfs_datanode_data_dir	path to dfs.datanode.data.dir	"{{ hadoop_datanode_db_dir }}/dfs/data"
hadoop_datanode_core_site_file	path to core-site.xml	"{{ hadoop_datanode_conf_dir }}/core-site.xml"
hadoop_datanode_hdfs_site_file	path to hdfs-site.xml	"{{ hadoop_datanode_conf_dir }}/hdfs-site.xml"
hadoop_datanode_mapred_site_file	path to mapred-site.xml	"{{ hadoop_datanode_conf_dir }}/mapred-site.xml"
hadoop_namenode_yarn_site_file	path to yarn-site.xml	"{{ hadoop_datanode_conf_dir }}/yarn-site.xml"
hadoop_config	a hash of xml config (see the example below)	{}

Dependencies

dict2xml is bundled in the role, with modifications, mostly removing six dependency. dict2xml is available at https://github.com/delfick/python-dict2xml.

Example Playbook

    - hosts: all
      pre_tasks:
        # XXX java.net.InetAddress.getLocalHost throws an exception without this
        - shell: echo "127.0.0.1 localhost {{ ansible_hostname }}" > /etc/hosts
          changed_when: False
      roles:
        - ansible-role-hadoop-namenode
        - ansible-role-hadoop-datanode
      vars:
        hadoop_config:
          slaves:
            - localhost
          core_site:
            - 
              - name: fs.defaultFS
              - value: 'hdfs://localhost/'
            -
              - name: hadoop.tmp.dir
              - value: /var/db/hadoop
            -
              - name: io.file.buffer.size
              - value: 131072
          hdfs_site:
            -
              - name: dfs.namenode.name.dir
              - value: "${hadoop.tmp.dir}/dfs/name"
            - 
              - name: dfs.blocksize
              - value: 268435456
            -
              - name: dfs.namenode.handler.count 
              - value: 100
            -
              - name: dfs.datanode.data.dir
              - value: "${hadoop.tmp.dir}/dfs/data"
          yarn_site:
            -
              - name: yarn.resourcemanager.scheduler.class
              - value: org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler
            -
              - name: yarn.acl.enable
              - value: false
            -
              - name: yarn.admin.acl
              - value: "*"
            -
              - name: yarn.log-aggregation-enable
              - value: false
            -
              - name: yarn.resourcemanager.hostname
              - value: localhost
            -
              - name: yarn.scheduler.capacity.root.queues
              - value: default
            -
              - name: yarn.scheduler.capacity.root.default.capacity
              - value: 100
            -
              - name: yarn.scheduler.capacity.root.default.state
              - value: RUNNING
            -
              - name: yarn.scheduler.capacity.root.default.user-limit-factor
              - value: 1
            -
              - name: yarn.scheduler.capacity.root.default.maximum-capacity
              - value: 100

License

BSD

Author Information

Tomoyuki Sakurai tomoyukis@reallyenglish.com