![Installing SPARK 2.0:
https://www.cloudera.com/documentation/spark2/latest/topics/spark2_installing.html
[root@mugzy ~]# wget http://archive.cloudera.com/spark2/csd/SPARK2_ON_YARN-2.3.0.cloudera2.jar
--2018-05-12 05:12:50-- http://archive.cloudera.com/spark2/csd/SPARK2_ON_YARN-2.3.0.cloudera2.jar
Resolving archive.cloudera.com (archive.cloudera.com)... 151.101.0.167, 151.101.64.167,
151.101.128.167, ...
Connecting to archive.cloudera.com (archive.cloudera.com)|151.101.0.167|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 19010 (19K) [application/java-archive]
Saving to: ‘SPARK2_ON_YARN-2.3.0.cloudera2.jar’
100%[======================================>] 19,010 13.9KB/s in 1.3s
2018-05-12 05:12:52 (13.9 KB/s) - ‘SPARK2_ON_YARN-2.3.0.cloudera2.jar’ saved [19010/19010]](https://image.slidesharecdn.com/installingspark2-180619101009/85/Installing-spark-2-1-320.jpg)
![[root@mugzy ~]# cp SPARK2_ON_YARN-2.3.0.cloudera2.jar /opt/cloudera/csd
[root@mugzy ~]# cd /opt/cloudera/csd
[root@mugzy csd]# ls
SPARK2_ON_YARN-2.3.0.cloudera2.jar
[root@mugzy csd]# chmod 644 SPARK2_ON_YARN-2.3.0.cloudera2.jar
[root@mugzy csd]# chown cloudera-scm:cloudera-scm SPARK2_ON_YARN-2.3.0.cloudera2.jar
[root@mugzy csd]# ls -l
total 20
-rw-r--r-- 1 cloudera-scm cloudera-scm 19010 May 12 05:16 SPARK2_ON_YARN-2.3.0.cloudera2.jar
[root@mugzy csd]#
[root@mugzy csd]# service cloudera-scm-server restart
Restarting cloudera-scm-server (via systemctl): [ OK ]
[root@mugzy csd]#](https://image.slidesharecdn.com/installingspark2-180619101009/85/Installing-spark-2-2-320.jpg)
![Installing Oracle Java JDK 1.8
https://tecadmin.net/install-java-8-on-centos-rhel-and-fedora/
[root@mugzy bin]# cd /opt/
[root@mugzy opt]# wget --no-cookies --no-check-certificate --header "Cookie: gpw
_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "
http://download.oracle.com/otn-pub/java/jdk/8u171-b11/512cd62ec5174c3487ac17c61a
aa89e8/jdk-8u171-linux-x64.tar.gz"
--2018-05-12 06:46:24-- http://download.oracle.com/otn-pub/java/jdk/8u171-b11/5
12cd62ec5174c3487ac17c61aaa89e8/jdk-8u171-linux-x64.tar.gz
Resolving download.oracle.com (download.oracle.com)... 23.35.212.173
Connecting to download.oracle.com (download.oracle.com)|23.35.212.173|:80... con
nected.
HTTP request sent, awaiting response... 302 Moved Temporarily
Location: https://edelivery.oracle.com/otn-pub/java/jdk/8u171-b11/512cd62ec5174c
3487ac17c61aaa89e8/jdk-8u171-linux-x64.tar.gz [following]](https://image.slidesharecdn.com/installingspark2-180619101009/85/Installing-spark-2-12-320.jpg)
![--2018-05-12 06:46:25-- https://edelivery.oracle.com/otn-pub/java/jdk/8u171-b11
/512cd62ec5174c3487ac17c61aaa89e8/jdk-8u171-linux-x64.tar.gz
Resolving edelivery.oracle.com (edelivery.oracle.com)... 23.50.186.175, 2a02:26f
0:c000:186::2d3e, 2a02:26f0:c000:183::2d3e
Connecting to edelivery.oracle.com (edelivery.oracle.com)|23.50.186.175|:443...
connected.
HTTP request sent, awaiting response... 302 Moved Temporarily
Location: http://download.oracle.com/otn-pub/java/jdk/8u171-b11/512cd62ec5174c34
87ac17c61aaa89e8/jdk-8u171-linux-x64.tar.gz?AuthParam=1526100506_26535ed17999858
17a2f3c9440bf8c35 [following]
--2018-05-12 06:46:26-- http://download.oracle.com/otn-pub/java/jdk/8u171-b11/5
12cd62ec5174c3487ac17c61aaa89e8/jdk-8u171-linux-x64.tar.gz?AuthParam=1526100506_
26535ed1799985817a2f3c9440bf8c35
Connecting to download.oracle.com (download.oracle.com)|23.35.212.173|:80... con
nected.
HTTP request sent, awaiting response... 200 OK
Length: 190890122 (182M) [application/x-gzip]
Saving to: ‘jdk-8u171-linux-x64.tar.gz’
100%[================================================================>] 190,890,122
106KB/s in 29m 40s
2018-05-12 07:16:07 (105 KB/s) - ‘jdk-8u171-linux-x64.tar.gz’ saved [190890122/190890122]
[root@mugzy opt]# ls
cloudera jdk-8u171-linux-x64.tar.gz
[root@mugzy opt]# scp jdk-8u171-linux-x64.tar.gz moly:/opt
ssh: Could not resolve hostname moly: Name or service not known
lost connection](https://image.slidesharecdn.com/installingspark2-180619101009/85/Installing-spark-2-13-320.jpg)
![[root@mugzy opt]# scp jdk-8u171-linux-x64.tar.gz molly:/opt
jdk-8u171-linux-x64.tar.gz 100% 182MB 60.7MB/s 00:03
[root@mugzy opt]# tar xzf jdk-8u171-linux-x64.tar.gz
[root@mugzy opt]# cd /opt/jdk1.8.0_171/
[root@mugzy jdk1.8.0_171]# cd /opt/jdk1.8.0_171/
[root@mugzy jdk1.8.0_171]# alternatives --install /usr/bin/java java /opt/jdk1.8.0_171/bin/java 2
[root@mugzy jdk1.8.0_171]# alternatives --config java
There is 1 program that provides 'java'.
Selection Command
-----------------------------------------------
*+ 1 /opt/jdk1.8.0_171/bin/java
Enter to keep the current selection[+], or type selection number:
[root@mugzy jdk1.8.0_171]# java -version
java version "1.8.0_171"
Java(TM) SE Runtime Environment (build 1.8.0_171-b11)
Java HotSpot(TM) 64-Bit Server VM (build 25.171-b11, mixed mode)
[root@mugzy jdk1.8.0_171]#](https://image.slidesharecdn.com/installingspark2-180619101009/85/Installing-spark-2-14-320.jpg)
![[root@mugzy conf]# alternatives --display spark2-conf
spark2-conf - status is auto.
link currently points to /etc/spark2/conf.cloudera.spark2_on_yarn
/opt/cloudera/parcels/SPARK2-2.3.0.cloudera2-1.cdh5.13.3.p0.316101/etc/spark2/conf.dist - priority 10
/etc/spark2/conf.cloudera.spark2_on_yarn - priority 51
Current `best' version is /etc/spark2/conf.cloudera.spark2_on_yarn.
[root@mugzy conf]# export JAVA_HOME=/opt/jdk1.8.0_171
[root@mugzy conf]# spark2-shell
Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
Welcome to
____ __
/ __/__ ___ _____/ /__
_ / _ / _ `/ __/ '_/
/___/ .__/_,_/_/ /_/_ version 2.3.0.cloudera2
/_/
Using Scala version 2.11.8 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_171)
Type in expressions to have them evaluated.
Type :help for more information.
scala>](https://image.slidesharecdn.com/installingspark2-180619101009/85/Installing-spark-2-18-320.jpg)
Installing spark 2
- 1. Installing SPARK 2.0: https://www.cloudera.com/documentation/spark2/latest/topics/spark2_installing.html [root@mugzy ~]# wget http://archive.cloudera.com/spark2/csd/SPARK2_ON_YARN-2.3.0.cloudera2.jar --2018-05-12 05:12:50-- http://archive.cloudera.com/spark2/csd/SPARK2_ON_YARN-2.3.0.cloudera2.jar Resolving archive.cloudera.com (archive.cloudera.com)... 151.101.0.167, 151.101.64.167, 151.101.128.167, ... Connecting to archive.cloudera.com (archive.cloudera.com)|151.101.0.167|:80... connected. HTTP request sent, awaiting response... 200 OK Length: 19010 (19K) [application/java-archive] Saving to: ‘SPARK2_ON_YARN-2.3.0.cloudera2.jar’ 100%[======================================>] 19,010 13.9KB/s in 1.3s 2018-05-12 05:12:52 (13.9 KB/s) - ‘SPARK2_ON_YARN-2.3.0.cloudera2.jar’ saved [19010/19010]
- 2. [root@mugzy ~]# cp SPARK2_ON_YARN-2.3.0.cloudera2.jar /opt/cloudera/csd [root@mugzy ~]# cd /opt/cloudera/csd [root@mugzy csd]# ls SPARK2_ON_YARN-2.3.0.cloudera2.jar [root@mugzy csd]# chmod 644 SPARK2_ON_YARN-2.3.0.cloudera2.jar [root@mugzy csd]# chown cloudera-scm:cloudera-scm SPARK2_ON_YARN-2.3.0.cloudera2.jar [root@mugzy csd]# ls -l total 20 -rw-r--r-- 1 cloudera-scm cloudera-scm 19010 May 12 05:16 SPARK2_ON_YARN-2.3.0.cloudera2.jar [root@mugzy csd]# [root@mugzy csd]# service cloudera-scm-server restart Restarting cloudera-scm-server (via systemctl): [ OK ] [root@mugzy csd]#
- 12. Installing Oracle Java JDK 1.8 https://tecadmin.net/install-java-8-on-centos-rhel-and-fedora/ [root@mugzy bin]# cd /opt/ [root@mugzy opt]# wget --no-cookies --no-check-certificate --header "Cookie: gpw _e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" " http://download.oracle.com/otn-pub/java/jdk/8u171-b11/512cd62ec5174c3487ac17c61a aa89e8/jdk-8u171-linux-x64.tar.gz" --2018-05-12 06:46:24-- http://download.oracle.com/otn-pub/java/jdk/8u171-b11/5 12cd62ec5174c3487ac17c61aaa89e8/jdk-8u171-linux-x64.tar.gz Resolving download.oracle.com (download.oracle.com)... 23.35.212.173 Connecting to download.oracle.com (download.oracle.com)|23.35.212.173|:80... con nected. HTTP request sent, awaiting response... 302 Moved Temporarily Location: https://edelivery.oracle.com/otn-pub/java/jdk/8u171-b11/512cd62ec5174c 3487ac17c61aaa89e8/jdk-8u171-linux-x64.tar.gz [following]
- 13. --2018-05-12 06:46:25-- https://edelivery.oracle.com/otn-pub/java/jdk/8u171-b11 /512cd62ec5174c3487ac17c61aaa89e8/jdk-8u171-linux-x64.tar.gz Resolving edelivery.oracle.com (edelivery.oracle.com)... 23.50.186.175, 2a02:26f 0:c000:186::2d3e, 2a02:26f0:c000:183::2d3e Connecting to edelivery.oracle.com (edelivery.oracle.com)|23.50.186.175|:443... connected. HTTP request sent, awaiting response... 302 Moved Temporarily Location: http://download.oracle.com/otn-pub/java/jdk/8u171-b11/512cd62ec5174c34 87ac17c61aaa89e8/jdk-8u171-linux-x64.tar.gz?AuthParam=1526100506_26535ed17999858 17a2f3c9440bf8c35 [following] --2018-05-12 06:46:26-- http://download.oracle.com/otn-pub/java/jdk/8u171-b11/5 12cd62ec5174c3487ac17c61aaa89e8/jdk-8u171-linux-x64.tar.gz?AuthParam=1526100506_ 26535ed1799985817a2f3c9440bf8c35 Connecting to download.oracle.com (download.oracle.com)|23.35.212.173|:80... con nected. HTTP request sent, awaiting response... 200 OK Length: 190890122 (182M) [application/x-gzip] Saving to: ‘jdk-8u171-linux-x64.tar.gz’ 100%[================================================================>] 190,890,122 106KB/s in 29m 40s 2018-05-12 07:16:07 (105 KB/s) - ‘jdk-8u171-linux-x64.tar.gz’ saved [190890122/190890122] [root@mugzy opt]# ls cloudera jdk-8u171-linux-x64.tar.gz [root@mugzy opt]# scp jdk-8u171-linux-x64.tar.gz moly:/opt ssh: Could not resolve hostname moly: Name or service not known lost connection
- 14. [root@mugzy opt]# scp jdk-8u171-linux-x64.tar.gz molly:/opt jdk-8u171-linux-x64.tar.gz 100% 182MB 60.7MB/s 00:03 [root@mugzy opt]# tar xzf jdk-8u171-linux-x64.tar.gz [root@mugzy opt]# cd /opt/jdk1.8.0_171/ [root@mugzy jdk1.8.0_171]# cd /opt/jdk1.8.0_171/ [root@mugzy jdk1.8.0_171]# alternatives --install /usr/bin/java java /opt/jdk1.8.0_171/bin/java 2 [root@mugzy jdk1.8.0_171]# alternatives --config java There is 1 program that provides 'java'. Selection Command ----------------------------------------------- *+ 1 /opt/jdk1.8.0_171/bin/java Enter to keep the current selection[+], or type selection number: [root@mugzy jdk1.8.0_171]# java -version java version "1.8.0_171" Java(TM) SE Runtime Environment (build 1.8.0_171-b11) Java HotSpot(TM) 64-Bit Server VM (build 25.171-b11, mixed mode) [root@mugzy jdk1.8.0_171]#
- 18. [root@mugzy conf]# alternatives --display spark2-conf spark2-conf - status is auto. link currently points to /etc/spark2/conf.cloudera.spark2_on_yarn /opt/cloudera/parcels/SPARK2-2.3.0.cloudera2-1.cdh5.13.3.p0.316101/etc/spark2/conf.dist - priority 10 /etc/spark2/conf.cloudera.spark2_on_yarn - priority 51 Current `best' version is /etc/spark2/conf.cloudera.spark2_on_yarn. [root@mugzy conf]# export JAVA_HOME=/opt/jdk1.8.0_171 [root@mugzy conf]# spark2-shell Setting default log level to "WARN". To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel). Welcome to ____ __ / __/__ ___ _____/ /__ _ / _ / _ `/ __/ '_/ /___/ .__/_,_/_/ /_/_ version 2.3.0.cloudera2 /_/ Using Scala version 2.11.8 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_171) Type in expressions to have them evaluated. Type :help for more information. scala>