Skip to content
GitLab
Commit 817c18c6 authored by Decoupes Remy's avatar Decoupes Remy
Browse files

Prepare /etc/hosts for all nodes using private network and ip static

parent d44b5186
master agile2020 inforsid2020
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 190 additions and 181 deletions
+190 -181
inventory/static
+ 5
− 5
View file @ 817c18c6
[master-nodes] [namenode]
namenode ansible_host=10.0.0.10
[slave-nodes] [datanodes]
datanode1 ansible_host=10.0.0.11
[edge-nodes] datanode2 ansible_host=10.0.0.11
[mono-node] [mono-node]
mononode-hdfs ansible_host=172.16.50.54 mononode-hdfs ansible_host=172.16.50.54
\ No newline at end of file
playbook/install-namenode.yml
+ 2
− 1
View file @ 817c18c6
...@@ -3,4 +3,5 @@ ...@@ -3,4 +3,5 @@
hosts: 'namenode' hosts: 'namenode'
roles: roles:
- common - common
\ No newline at end of file - hadoop
\ No newline at end of file
playbook/roles/hadoop/tasks/main.yml
+ 163
− 162
View file @ 817c18c6
...@@ -4,167 +4,168 @@ ...@@ -4,167 +4,168 @@
msg: msg:
- "Hadoop only support java jdk 8, see https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions when it'll support jdk 11" - "Hadoop only support java jdk 8, see https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions when it'll support jdk 11"
- "Be aware that things may not work when using jdk 11 like explore HDFS using webserver on port 9870" - "Be aware that things may not work when using jdk 11 like explore HDFS using webserver on port 9870"
- "Default IPv4 address is : {{ ansible_default_ipv4.address }}"
- name: Set java home as environment variable
become: yes # - name: Set java home as environment variable
apt: # become: yes
name: # apt:
- openjdk-11-jdk # name:
# - openjdk-11-jdk
- name: create hadoop group
become: yes # - name: create hadoop group
group: # become: yes
name: hadoop # group:
# name: hadoop
- name: create hadoop user
become: yes # - name: create hadoop user
user: # become: yes
name: hadoop # user:
group: hadoop # name: hadoop
home: "{{ hadoopUSRHome }}" # group: hadoop
createhome: yes # home: "{{ hadoopUSRHome }}"
system: yes # createhome: yes
# system: yes
- name: Set JAVA_HOME as environment variable
become: yes # - name: Set JAVA_HOME as environment variable
become_user : hadoop # become: yes
blockinfile: # become_user : hadoop
insertafter: EOF # blockinfile:
path : ~/.bashrc # insertafter: EOF
block: | # path : ~/.bashrc
export JAVA_HOME={{ javahome }} # block: |
export HADOOP_HOME=/usr/local/hadoop # export JAVA_HOME={{ javahome }}
export PATH=$PATH:$HADOOP_HOME/bin # export HADOOP_HOME=/usr/local/hadoop
# export PATH=$PATH:$HADOOP_HOME/bin
- name: source .bashrc
become: yes # - name: source .bashrc
become_user: hadoop # become: yes
shell: source ~/.bashrc # become_user: hadoop
args: # shell: source ~/.bashrc
executable: /bin/bash # args:
# executable: /bin/bash
- name: create .ssh directory
become: yes # - name: create .ssh directory
file: # become: yes
path: "{{ hadoopUSRHome }}/.ssh/" # file:
state: directory # path: "{{ hadoopUSRHome }}/.ssh/"
owner: hadoop # state: directory
group: hadoop # owner: hadoop
mode: 0700 # group: hadoop
# mode: 0700
- name: copy ssh key
become: yes # - name: copy ssh key
copy: # become: yes
src: "{{ item }}" # copy:
dest: "{{ hadoopUSRHome }}/.ssh/" # src: "{{ item }}"
owner: hadoop # dest: "{{ hadoopUSRHome }}/.ssh/"
group: hadoop # owner: hadoop
mode: 0600 # group: hadoop
with_items: # mode: 0600
- keys/id_rsa # with_items:
- keys/id_rsa.pub # - keys/id_rsa
# - keys/id_rsa.pub
- name: authorized ssh key for hadoop user
become: yes # - name: authorized ssh key for hadoop user
authorized_key: # become: yes
user: hadoop # authorized_key:
state: present # user: hadoop
key: "{{ lookup('file', 'keys/id_rsa.pub') }}" # state: present
# key: "{{ lookup('file', 'keys/id_rsa.pub') }}"
- name: create a tempory directory # - name: create a tempory directory
become: yes # become: yes
file: # file:
state: directory # state: directory
path: "{{ hadoopUSRHome }}/tmp" # path: "{{ hadoopUSRHome }}/tmp"
- name: create a prod directory for hadoop # - name: create a prod directory for hadoop
become: yes # become: yes
file: # file:
state: directory # state: directory
path: "{{ hadoopDir }}" # path: "{{ hadoopDir }}"
- name: "Download and Extract hadoop-{{ hadoopVersion }}" # - name: "Download and Extract hadoop-{{ hadoopVersion }}"
become: yes # become: yes
unarchive: # unarchive:
src: "http://apache.mirrors.ovh.net/ftp.apache.org/dist/hadoop/core/hadoop-{{ hadoopVersion }}/hadoop-{{ hadoopVersion }}.tar.gz" # src: "http://apache.mirrors.ovh.net/ftp.apache.org/dist/hadoop/core/hadoop-{{ hadoopVersion }}/hadoop-{{ hadoopVersion }}.tar.gz"
remote_src: yes # remote_src: yes
dest: "{{ hadoopDir }}" # dest: "{{ hadoopDir }}"
extra_opts: [--strip-components=1] # extra_opts: [--strip-components=1]
owner: hadoop # owner: hadoop
group: hadoop # group: hadoop
- name : Set JAVA_HOME in hadoop-env.sh # - name : Set JAVA_HOME in hadoop-env.sh
become: yes # become: yes
blockinfile: # blockinfile:
insertafter: EOF # insertafter: EOF
path: "{{ hadoopDir }}/etc/hadoop/hadoop-env.sh" # path: "{{ hadoopDir }}/etc/hadoop/hadoop-env.sh"
block: "export JAVA_HOME={{ javahome }}" # block: "export JAVA_HOME={{ javahome }}"
- name: configure core-site.xml # - name: configure core-site.xml
become: yes # become: yes
template: # template:
src: templates/core-site.j2 # src: templates/core-site.j2
dest: "{{ hadoopDir }}/etc/hadoop/core-site.xml" # dest: "{{ hadoopDir }}/etc/hadoop/core-site.xml"
owner: hadoop # owner: hadoop
group: hadoop # group: hadoop
- name: configure hdfs-site.xml # - name: configure hdfs-site.xml
become: yes # become: yes
template: # template:
src: templates/hdfs-site.j2 # src: templates/hdfs-site.j2
dest: "{{ hadoopDir }}/etc/hadoop/hdfs-site.xml" # dest: "{{ hadoopDir }}/etc/hadoop/hdfs-site.xml"
owner: hadoop # owner: hadoop
group: hadoop # group: hadoop
- name: configure mapred-site.xml # - name: configure mapred-site.xml
become: yes # become: yes
template: # template:
src: templates/mapred-site.j2 # src: templates/mapred-site.j2
dest: "{{ hadoopDir }}/etc/hadoop/mapred-site.xml" # dest: "{{ hadoopDir }}/etc/hadoop/mapred-site.xml"
owner: hadoop # owner: hadoop
group: hadoop # group: hadoop
- name: copy hadoop service file # - name: copy hadoop service file
become: yes # become: yes
template: # template:
src: templates/hadoop.service.j2 # src: templates/hadoop.service.j2
dest: /etc/systemd/system/hadoop.service # dest: /etc/systemd/system/hadoop.service
- name: enable hadoop service # - name: enable hadoop service
become: yes # become: yes
service: # service:
daemon_reload: yes # daemon_reload: yes
name: hadoop # name: hadoop
state: stopped # state: stopped
enabled: yes # enabled: yes
- name: HDFS has been already formatted ? # - name: HDFS has been already formatted ?
become: yes # become: yes
stat: # stat:
path: /tmp/hadoop-hadoop/dfs/name/current/VERSION # path: /tmp/hadoop-hadoop/dfs/name/current/VERSION
register: file_exist # register: file_exist
- debug: # - debug:
msg: "/tmp/hadoop-hadoop/dfs/name/current/VERSION exists ? : {{ file_exist.stat.exists}}" # msg: "/tmp/hadoop-hadoop/dfs/name/current/VERSION exists ? : {{ file_exist.stat.exists}}"
- name: format HDFS # - name: format HDFS
become: yes # become: yes
become_user: hadoop # become_user: hadoop
shell: "{{ hadoopDir }}/bin/hdfs namenode -format" # shell: "{{ hadoopDir }}/bin/hdfs namenode -format"
args: # args:
executable: /bin/bash # executable: /bin/bash
when: file_exist.stat.exists == False # when: file_exist.stat.exists == False
- name: stopped hadoop service # - name: stopped hadoop service
become: yes # become: yes
service: # service:
name: hadoop # name: hadoop
state: stopped # state: stopped
- name: start hadoop service # - name: start hadoop service
become: yes # become: yes
service: # service:
name: hadoop # name: hadoop
state: started # state: started
vagrant/cluster/Vagrantfile
+ 20
− 13
View file @ 817c18c6
Vagrant.configure("2") do |config| Vagrant.configure("2") do |config|
config.vm.box = "generic/debian10"
config.vm.network "public_network", bridge:"enp1s0", use_dhcp_assigned_default_route: true
config.vm.provision "shell", inline: <<-SHELL
systemctl stop resolvconf
echo "nameserver 10.34.192.61" > /etc/resolv.conf
echo "nameserver 10.34.192.62" >> /etc/resolv.conf
SHELL
config.vm.define "namenode" do |namenode| config.vm.define "namenode" do |namenode|
config.vm.hostname = "namenode" namenode.vm.box = "generic/debian10"
config.vm.provision "ansible" do |ansible| namenode.vm.network "public_network", bridge:"enp1s0", use_dhcp_assigned_default_route: true
namenode.vm.provision "shell", inline: <<-SHELL
systemctl stop resolvconf
echo "nameserver 10.34.192.61" > /etc/resolv.conf
echo "nameserver 10.34.192.62" >> /etc/resolv.conf
SHELL
namenode.vm.hostname = "namenode"
namenode.vm.network :private_network, ip: "10.0.0.10"
namenode.vm.provision "ansible" do |ansible|
ansible.playbook = "/home/rdecoupe/Documents/TETIS/projet/aidmoit/ansible-deployment/playbook/install-namenode.yml" ansible.playbook = "/home/rdecoupe/Documents/TETIS/projet/aidmoit/ansible-deployment/playbook/install-namenode.yml"
end end
end end
...@@ -18,10 +18,17 @@ Vagrant.configure("2") do |config| ...@@ -18,10 +18,17 @@ Vagrant.configure("2") do |config|
N = 2 N = 2
(1..2).each do |machine_id| (1..2).each do |machine_id|
config.vm.define "datanode#{machine_id}" do |machine| config.vm.define "datanode#{machine_id}" do |machine|
config.vm.hostname = "datanode#{machine_id}" machine.vm.box = "generic/debian10"
machine.vm.network "public_network", bridge:"enp1s0", use_dhcp_assigned_default_route: true
machine.vm.provision "shell", inline: <<-SHELL
systemctl stop resolvconf
echo "nameserver 10.34.192.61" > /etc/resolv.conf
echo "nameserver 10.34.192.62" >> /etc/resolv.conf
SHELL
machine.vm.hostname = "datanode#{machine_id}"
machine.vm.network :private_network, ip: "10.0.0.1#{machine_id}"
if machine_id == N if machine_id == N
config.vm.provision "ansible" do |ansible| machine.vm.provision "ansible" do |ansible|
# ansible.playbook = "/home/rdecoupe/Documents/TETIS/projet/aidmoit/ansible-deployment/playbook/install-datanode.yml"
ansible.playbook = "../../playbook/install-datanode.yml" ansible.playbook = "../../playbook/install-datanode.yml"
end end
end end
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment