Commit 0eedfc72 authored by Decoupes Remy's avatar Decoupes Remy
Browse files

prepare to push

No related merge requests found
Showing with 327 additions and 20 deletions
+327 -20
ansible-playbook -i inventory/static playbook/mononode.yml
#ssh-copy-id vagrant@172.16.50.54
ansible-playbook -i inventory/static playbook/mononode.yml -u vagrant --become --ask-become-pass
......@@ -5,4 +5,5 @@
[edge-nodes]
[mono-node]
mononode ansible_host=172.16.50.54 ansible_user=vagrant ansible_ssh_pass=vagrant
\ No newline at end of file
#mononode ansible_host=172.16.50.54 ansible_user=vagrant ansible_ssh_pass=vagrant
mononode-hdfs ansible_host=172.16.50.54
\ No newline at end of file
---
- name: deploy HDFS on single node
hosts: 'mono-node'
hosts: 'mononode-hdfs'
tasks:
- name: Full upgrade for the system
become: yes
apt:
update_cache: yes
upgrade: yes
autoclean: yes
autoremove: yes
roles:
- common
- hadoop
tasks:
- name: Full upgrade for the system
become: yes
apt:
update_cache: yes
upgrade: yes
autoclean: yes
autoremove: yes
- name: Full upgrade for the system
become: yes
apt:
update_cache: yes
upgrade: yes
autoclean: yes
autoremove: yes
- name: Install dependencies
become: yes
apt:
name:
# - maven
# - python-dev
# - python-pip
- unzip
- tree
- htop
- vim
- name: Prepare Download directory
become: yes
file:
state: directory
path: /root/ansible-dl
-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn
NhAAAAAwEAAQAAAgEAzoRd7fNHHwJYux4+qtWjMOgo3I3OhNCQ9xYrnHqFdYzD9t6BXr4D
9JK8JHfdO0n7sKWae735P7Z5RLa92XMUtYiF4z4f+K7+5nNuTeo6/yFFBRLoHoyFVFA4km
cKtF/RJPoks9DtlkEzPHE1hWVK69861L37p+g0J/NcCTCKrUPCw4HCdpxBo8n8IoGP/An7
oMGLfDLW2RToK9lIZW/k1y2nKFe5krx1SjFMKmjalny7x1Uyce9ss27JaTkzH0mwM/8vnA
LpN4MBM1PEgr0BHXdC6YpM4buu6xCASbQ1+Q/JOrH2U3TU16ITI9zHzfC8CEvC97Ao53XB
4Ji4NXAXUMYHE1vt0kbXwX3GDONo4lF7qI7Hsa1ae28aJ5uAXz9T8HLZ/++DZcnHg9WMuG
av4FxB+Wiw5hWrkZeBYqjbY+R6lJKLyHq3QbsDVYhfoO7Bv+9fX623hJ8KAyGK17XO4GBA
4Lc9/+d793t64GV+upYa0wAlNE6kYQ68aV8eWyIhni16GaFgWK8YI21+HSBIP24rCSX6qG
rFTfL77sSILulgBcszKxEeLeukQdbV4ZKLEMg6APpkUYGrmC1GuqANg8KOtJVFrf8sY2wP
9+dFgSaXAERKcXrcL3Stqys1pgw+/FbPs6Bz9vcb8ZQc8j/VGzaqhxbbxfEguGgfv1djVN
sAAAc4V1YUBVdWFAUAAAAHc3NoLXJzYQAAAgEAzoRd7fNHHwJYux4+qtWjMOgo3I3OhNCQ
9xYrnHqFdYzD9t6BXr4D9JK8JHfdO0n7sKWae735P7Z5RLa92XMUtYiF4z4f+K7+5nNuTe
o6/yFFBRLoHoyFVFA4kmcKtF/RJPoks9DtlkEzPHE1hWVK69861L37p+g0J/NcCTCKrUPC
w4HCdpxBo8n8IoGP/An7oMGLfDLW2RToK9lIZW/k1y2nKFe5krx1SjFMKmjalny7x1Uyce
9ss27JaTkzH0mwM/8vnALpN4MBM1PEgr0BHXdC6YpM4buu6xCASbQ1+Q/JOrH2U3TU16IT
I9zHzfC8CEvC97Ao53XB4Ji4NXAXUMYHE1vt0kbXwX3GDONo4lF7qI7Hsa1ae28aJ5uAXz
9T8HLZ/++DZcnHg9WMuGav4FxB+Wiw5hWrkZeBYqjbY+R6lJKLyHq3QbsDVYhfoO7Bv+9f
X623hJ8KAyGK17XO4GBA4Lc9/+d793t64GV+upYa0wAlNE6kYQ68aV8eWyIhni16GaFgWK
8YI21+HSBIP24rCSX6qGrFTfL77sSILulgBcszKxEeLeukQdbV4ZKLEMg6APpkUYGrmC1G
uqANg8KOtJVFrf8sY2wP9+dFgSaXAERKcXrcL3Stqys1pgw+/FbPs6Bz9vcb8ZQc8j/VGz
aqhxbbxfEguGgfv1djVNsAAAADAQABAAACAE2g7hny98oNPMNhtBbNZaGNs0Zsu7OrHsLy
f4Q7fg1Y05TpwOGXNa9qtPRUZkHSJp0ZMv/OwnGTqvQMaULjAmbWodVw1d3KEiEYRN9e67
xZijNFDyHkmn2+BfXqa9ivp9NpgABjydtQOO1AdOjm0YnmiJvNqlpMa8aUsQT3TTmiHgUV
sr6ZMAdcdr2VzETmj8K23jLafW5ucHODIraWoAPOsmX0y9n/o2s3KrVKYthsnL1qJiMz57
1vKKC4EqJxNZ3mD8RqR7ZJNVtQ4q5uU4v695kIFqxHu5G6yy5pYlxz9aVtnfVzFkrULrOt
WyHrmxLXP7uRaV7QIgfGAIB92RUjIWO/Fn1EAKXsTmKiKOESIIN8EbXGNi6ZbeQASLRXiR
qvwF5THCJUNKn/xoya0KslaEPaW2jt5vtvhqaftNXjbO7fcoTZe4i78tmJKbZkm5DFu82r
UH6mQqZiaPXsz31UWjRsDseQG3P0Hbtjdp6NbrAy8PqWkCPr7Kr+qVBSS8yyYWpq1UF0VB
w8ewv7nl7vdN+OlByMTT6Prrkayv01+MAkAJy3geEBt05spv+mD0IiQm+YZWKHfSNa9vEg
MJkdeoW/6Aq3n5MBFYUMzr8AuKUhdQMLDcV27EK40WaOyvYYDNvDb5RMvU9mkMXOu9Rfxf
kWax092FnnOGq5w2VBAAABADF7MIGupMkTdCNAhib5pkMuqEtLCS4zBtqmjCUtxTsQwo7E
iQdm9+lvnUlTU32levyEbHYb/Yy/v1QMPcPmuobj/rUZH2lVu3M6sjhBM6POsKaOJHnl5v
cbunRckqleFPgNMdhRo+X7I4JyFS3xshNazDPdhGeqmL01EfYWAnm07QM3eA2UNuzLp26D
878lr777YBr3wYVnwP4wGopeNul9OWQLqtba7K4h5B/CJRVzpPc+1aSWuo+MMXzvLwramo
QsgRftXDlaNXkKA3004wP4YmqgZn59/vE/c1UfYQrH5tHzlk6yScTKcLLiSNvI9kkpXb2p
0x1w08w6cRN0MgIAAAEBAP0nNi+No2yWopTxg0Z2/7oksxwNEKazDWjlskeGSCVIleIwNK
Gg4swPfdS/UAMMGeL0MFNMKr45Fr61GrSG5ioF8yGFtmuOGxoYUTAI8fr4D6PjeLw8MYjm
zuNs9Hu+AsuDoO01L8FNWes0CcTIOAChaE+wtpzLPMI9L+rPB1kiWL0hM34l/UiolUaWVJ
d8MGUxg4CLecg+jRYqaiU8B2ksrDAeqAVAoPQqoYt7Qd+Pa28xLk1elUeBtym3LyYoQLQg
qyIbgZ3wClYwgM5WETs9gaFfu/cg/ws4QM92acLSzAhkTfvf+dYH4kcx04QtnIoUdctEmZ
ZWOWOlIb0iwbsAAAEBANDW5a3f92Be7MprvNlWEb0DlGTSqAKSDlS4OzHn0KoHAfooz7SS
LdvM40z1JEna5LPbXrUciZ0tMdxMkLapKsS+yGgP3enfC5g+yjwEwnxDQeaIaNlUZcX5Ac
pMJecIH+m6QjqAfPGe4GhYN+8OVzql7aZKQ5F1xWlOnOD+YD+1/Smeu8Hxr0v7o+KjpoFF
Ey7iu526IuX4hfk0pZmnX23uri0Yw9kN2z3IP5mOxC8STD5qVHheJhbF2ZKG4zjzPvtlXi
tJkhZJoeoGWYI/Qgj4Os7/POWve47B5E3svKdZFFo8QxuvNRsovOKRWj27L547TB5lFAyB
B4G8x7/ed2EAAAAAAQID
-----END OPENSSH PRIVATE KEY-----
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDOhF3t80cfAli7Hj6q1aMw6Cjcjc6E0JD3FiuceoV1jMP23oFevgP0krwkd907SfuwpZp7vfk/tnlEtr3ZcxS1iIXjPh/4rv7mc25N6jr/IUUFEugejIVUUDiSZwq0X9Ek+iSz0O2WQTM8cTWFZUrr3zrUvfun6DQn81wJMIqtQ8LDgcJ2nEGjyfwigY/8CfugwYt8MtbZFOgr2Uhlb+TXLacoV7mSvHVKMUwqaNqWfLvHVTJx72yzbslpOTMfSbAz/y+cAuk3gwEzU8SCvQEdd0Lpikzhu67rEIBJtDX5D8k6sfZTdNTXohMj3MfN8LwIS8L3sCjndcHgmLg1cBdQxgcTW+3SRtfBfcYM42jiUXuojsexrVp7bxonm4BfP1Pwctn/74NlyceD1Yy4Zq/gXEH5aLDmFauRl4FiqNtj5HqUkovIerdBuwNViF+g7sG/719frbeEnwoDIYrXtc7gYEDgtz3/53v3e3rgZX66lhrTACU0TqRhDrxpXx5bIiGeLXoZoWBYrxgjbX4dIEg/bisJJfqoasVN8vvuxIgu6WAFyzMrER4t66RB1tXhkosQyDoA+mRRgauYLUa6oA2Dwo60lUWt/yxjbA/350WBJpcAREpxetwvdK2rKzWmDD78Vs+zoHP29xvxlBzyP9UbNqqHFtvF8SC4aB+/V2NU2w==
# Playbook based on https://fr.blog.businessdecision.com/tutoriel-cluster-hadoop/
- debug:
msg:
- "Hadoop only support java jdk 8, see https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions when it'll support jdk 11"
- "Be aware that things may not work when using jdk 11 like explore HDFS using webserver on port 9870"
- name: Set java home as environment variable
become: yes
apt:
name:
- openjdk-11-jdk
- name: create hadoop group
become: yes
group:
name: hadoop
- name: create hadoop user
become: yes
user:
name: hadoop
group: hadoop
home: "{{ hadoopUSRHome }}"
createhome: yes
system: yes
- name: Set JAVA_HOME as environment variable
become: yes
become_user : hadoop
blockinfile:
insertafter: EOF
path : ~/.bashrc
block: |
export JAVA_HOME={{ javahome }}
export HADOOP_HOME=/usr/local/hadoop
export PATH=$PATH:$HADOOP_HOME/bin
- name: source .bashrc
become: yes
become_user: hadoop
shell: source ~/.bashrc
args:
executable: /bin/bash
- name: create .ssh directory
become: yes
file:
path: "{{ hadoopUSRHome }}/.ssh/"
state: directory
owner: hadoop
group: hadoop
mode: 0700
- name: copy ssh key
become: yes
copy:
src: "{{ item }}"
dest: "{{ hadoopUSRHome }}/.ssh/"
owner: hadoop
group: hadoop
mode: 0600
with_items:
- keys/id_rsa
- keys/id_rsa.pub
- name: authorized ssh key for hadoop user
become: yes
authorized_key:
user: hadoop
state: present
key: "{{ lookup('file', 'keys/id_rsa.pub') }}"
- name: create a tempory directory
become: yes
file:
state: directory
path: "{{ hadoopUSRHome }}/tmp"
- name: create a prod directory for hadoop
become: yes
file:
state: directory
path: "{{ hadoopDir }}"
- name: "Download and Extract hadoop-{{ hadoopVersion }}"
become: yes
unarchive:
src: "http://apache.mirrors.ovh.net/ftp.apache.org/dist/hadoop/core/hadoop-{{ hadoopVersion }}/hadoop-{{ hadoopVersion }}.tar.gz"
remote_src: yes
dest: "{{ hadoopDir }}"
extra_opts: [--strip-components=1]
owner: hadoop
group: hadoop
- name : Set JAVA_HOME in hadoop-env.sh
become: yes
blockinfile:
insertafter: EOF
path: "{{ hadoopDir }}/etc/hadoop/hadoop-env.sh"
block: "export JAVA_HOME={{ javahome }}"
- name: configure core-site.xml
become: yes
template:
src: templates/core-site.j2
dest: "{{ hadoopDir }}/etc/hadoop/core-site.xml"
owner: hadoop
group: hadoop
- name: configure hdfs-site.xml
become: yes
template:
src: templates/hdfs-site.j2
dest: "{{ hadoopDir }}/etc/hadoop/hdfs-site.xml"
owner: hadoop
group: hadoop
- name: configure mapred-site.xml
become: yes
template:
src: templates/mapred-site.j2
dest: "{{ hadoopDir }}/etc/hadoop/mapred-site.xml"
owner: hadoop
group: hadoop
- name: copy hadoop service file
become: yes
template:
src: templates/hadoop.service.j2
dest: /etc/systemd/system/hadoop.service
- name: enable hadoop service
become: yes
service:
daemon_reload: yes
name: hadoop
state: stopped
enabled: yes
- name: HDFS has been already formatted ?
become: yes
stat:
path: /tmp/hadoop-hadoop/dfs/name/current/VERSION
register: file_exist
- debug:
msg: "/tmp/hadoop-hadoop/dfs/name/current/VERSION exists ? : {{ file_exist.stat.exists}}"
- name: format HDFS
become: yes
become_user: hadoop
shell: "{{ hadoopDir }}/bin/hdfs namenode -format"
args:
executable: /bin/bash
when: file_exist.stat.exists == False
- name: stopped hadoop service
become: yes
service:
name: hadoop
state: stopped
- name: start hadoop service
become: yes
service:
name: hadoop
state: started
<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://localhost:9000</value>
</property>
</configuration>
\ No newline at end of file
[Unit]
Description=Hadoop DFS namenode and datanode
After=syslog.target network.target remote-fs.target nss-lookup.target network-online.target
Requires=network-online.target
[Service]
User=hadoop
Group=hadoop
Type=forking
ExecStart={{ hadoopDir }}sbin/start-dfs.sh
ExecStop={{ hadoopDir }}sbin/stop-dfs.sh
WorkingDirectory={{ hadoopDir }}
Environment=JAVA_HOME={{ javahome }}
Environment=HADOOP_HOME={{ hadoopDir }}
TimeoutStartSec=2min
# Restart=on-failure
PIDFile=/tmp/hadoop-hadoop-namenode.pid
[Install]
WantedBy=multi-user.target
\ No newline at end of file
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
</configuration>
<property>
<name>mapred.job.tracker</name>
<value>localhost:9001</value>
</property>
\ No newline at end of file
hadoopVersion: 3.2.1
hadoopUSRHome: /home/hadoop
hadoopDir: /usr/local/hadoop/
javahome: /usr/lib/jvm/java-11-openjdk-amd64
openjdk8URL : http://security-cdn.debian.org/debian-security/pool/updates/main/o/openjdk-8/openjdk-8-jdk_8u232-b09-1~deb9u1_amd64.deb
\ No newline at end of file
1.5:f20d4edf-e217-4543-b69e-0cc7dbe78913
\ No newline at end of file
1571829768
\ No newline at end of file
1000
\ No newline at end of file
f20d4edf-e217-4543-b69e-0cc7dbe78913
\ No newline at end of file
8120b87d0a9c45f8b554d05cc4ab81d8
\ No newline at end of file
-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEAugjlJXw+jX0upTiMX7rEgqW3KQ2JQ+DvWJ4XRCufn6yxsVKf
cIMRrDGbCKfwpiD/GXclXWYwh4+TZgNS7ugsxyAgln72XCVAgdWtyGQnvtmmuc4L
grbh9sBneWTGdJnz8RH12Lvv3E24f+6mPVWqChyAk48fymzJWkUDclmDSFubHtUE
edqKLioMX5jVddSZzlfMqsS/cmZOTVdzef8BD/WEW1gwohMVsqUiHr9mKCTNnWzK
o70FscWcrAjKWTgHf2i4DChYwxiVgM445CoH8p6RWeZKvz7mOvK7jTHbiXKTygcw
GA0KfvUkehY9sTAU+meI1oJrUFHRqVeVXBMp1QIDAQABAoIBAGk+pwQx6eF5tG3c
M+3kZ0/SGHCtK76u/jkQFvGGDGP0+wHqUDECtXjJB4A6IjaZ7Den5AeWzFWksCrC
LqThzJNKZaU+Znlyya+qBwqbPrEFyQDMxE5K0EMcTm2qyWslvY/gLUlT6Tqk7X/S
r1F7qRrXTRYtnjqSmMbszCpg4ZiI4lD7SSE2byHLyDu3UoW8g4NVQvs/+jJHHAvJ
kBRSFylSf6u5LQqiM4mrt1gkJ2BkiaJTxBauPskjLBEsNpo+NyIW56drqLbKx8/Z
ZwrAzZCrSIhpUOM7Ufc8k2GQboZ1q/0Kr9zMc42btPQpmGE+TIXj5NVyWnRgTUm7
Mdb4LwECgYEA8jZ2M8evmAca9W1qvgrkUU3ZyZfzGGfAI5vYmvWFE7iKdwo1zfqU
Qhy6oqGLqqK72kfTye/T0wIA/jc1p63wqxhDWw+pxNwiuMarP7xm5tijbTBPGKO+
FPV9kozUTU0ZxNMiA1gst2m+HvUrS+qs8qxndF2KkXVjmoHHgT9CQGUCgYEAxJ/N
6WJE58QpulfZM/fpkCyUna5LDi7LE8b4z9wYoQ/fpwBCnIRssYhHA7OyngU0Eukg
3wvHPoMDB4tG6mRuNhfko9coizaBpr3URFVAbyuKcpo1OOzvq/DCqyCK9eW0dXXX
TckLKLqTHPDj5dohfSRgLNK8P/+zLqlXRPSd1LECgYACoLKhly4BgTo6pWg+DZIy
i5ysQjjB8AY2nK855yX9c8gUyIiS5oALQ5KGxyfjvwY4tQTIMRDkPiZSm7i6S/nL
cWTxrPbTiuvZg1ZCR8H6CSmkupxxEP2z7oHhQCPhfs+jwgKhdKhviPcmENTaav5b
V6F7srQZJePhvjyy20PVvQKBgQCHWeJrOARq7vQynGqehZg4cRfU+7gbCUcuVBfP
Lpmdtcmlsl4oVFhmVYNdHwRkFvzjz6ubHwaHF7SsKsokbAIaJpRz3JbCTkEtwRsv
9uemH3jsrw7YIvJRkCjRmFNQNlHlUbtmhnCyFs1SbNjUzzWiabny4RZJkAGOq2+x
vhov4QKBgAonGk9WobsFr6WawmrlK9Q1yhdQBrLRmt9JN8p09N2Iv3gRYIQnv1QM
xP7D4darqyFuTIil2YIcwkicrTM7Qw+OnZjd+BKkfL8cbvoHrgmS6OMlUbOl+cS5
cuyKW5WxhRXtNAzoySfDdT35rIl4ph6EKH03RPq5zB5uv8tXpvWp
-----END RSA PRIVATE KEY-----
{}
\ No newline at end of file
/home/rdecoupe/Documents/TETIS/projet/aidmoit/ansible-deployment/vagrant/mononode
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment