I am not going to write too much about Cloudera as there is tons of information available in the world of Google.
What I am going to write here is the simple steps to setup Cloudera in Single User Mode on Oracle Linux 7.
You can download Oracle Linux 7 from https://edelivery.oracle.com/osdc/faces/Home.jspx (you need to have metalink access).
Install Oracle Linux with Server with GUI as Base Environment and Compatibility Libraries and Development Tools as Add-Ons.
I have spent couple of long hours to install Cloudera Manager with so many different errors and what not. So, Finally came out with these simple steps and scripts to setup the CDH 5 environment up and running in probably about an hour.
Used laptop configuration:
i5 processor with 16GB RAM (good luck if you have anything less than this)
OS: OL7
Steps:
01] Disable SELinux - edit /etc/selinux/config file and change the value for SELINUX to disabled and reboot and make sure "getenforce" command returns Disabled.
02] Add cloudera-scm user group to SUDO list:
#visudo
and then add the below line
%cloudera-scm ALL=(ALL) NOPASSWD:ALL
Also add below lines - believe me you are going to hit some errors if you dont.
Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin
Defaults env_keep +="JAVA_HOME"
comment this line 'Defaults requiretty'
03] Add following line to /etc/pam.d/su file
session required pam_limits.so
04] Disable ipv6:
edit /etc/sysctl.conf and add the below:
# to disable IPv6 on all interfaces system wide
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
# to disable IPv6 on a specific interface (e.g., eth0, lo)
net.ipv6.conf.enp0s25.disable_ipv6 = 1
05] Make sure your /etc/host file has only this line and nothing else (because this is Single User Mode and just 127.0.0.1 is enough)
127.0.0.1 localhost
06] Now is the fun part. This is where I had to spend way too much time figuring out what causing the issue. Even though we added cloudera-scm into SUDOers list, and logged in as root, the installation part was not able to create any directories even after providing root password and password less SSH. So ended up in capturing every single folder that is needed with appropriate mode and listed in the below script. Run this script to create all required folders.
======Begin=====
mkdir -vp /var/lib/zookeeper
chown cloudera-scm:cloudera-scm /var/lib/zookeeper
chmod 775 /var/lib/zookeeper
mkdir -vp /var/log/zookeeper
chown cloudera-scm:cloudera-scm /var/log/zookeeper
chmod 775 /var/log/zookeeper
mkdir -vp /var/lib/zookeeper/version-2
chown cloudera-scm:cloudera-scm /var/lib/zookeeper/version-2
chmod 775 /var/lib/zookeeper/version-2
mkdir /cloudera_manager_zookeeper_canary
chown cloudera-scm:cloudera-scm /cloudera_manager_zookeeper_canary
chmod 775 /cloudera_manager_zookeeper_canary
mkdir -vp /var/lib/cloudera-scm-eventserver
chown cloudera-scm:cloudera-scm /var/lib/cloudera-scm-eventserver
chmod 775 /var/lib/cloudera-scm-eventserver
mkdir -vp /var/log/cloudera-scm-eventserver
chown cloudera-scm:cloudera-scm /var/log/cloudera-scm-eventserver
chmod 775 /var/log/cloudera-scm-eventserver
mkdir -vp /var/lib/cloudera-host-monitor /var/log/cloudera-host-monitor
chown cloudera-scm:cloudera-scm /var/lib/cloudera-host-monitor/ /var/log/cloudera-host-monitor/
chmod 775 /var/lib/cloudera-host-monitor/ /var/log/cloudera-host-monitor/
mkdir -vp /var/lib/cloudera-scm-firehose /var/log/cloudera-scm-firehose
chown cloudera-scm:cloudera-scm /var/lib/cloudera-scm-firehose /var/log/cloudera-scm-firehose
chmod 775 /var/lib/cloudera-scm-firehose /var/log/cloudera-scm-firehose
mkdir -vp /var/lib/cloudera-scm-alertpublisher /var/log/cloudera-scm-alertpublisher
chown cloudera-scm:cloudera-scm /var/lib/cloudera-scm-alertpublisher /var/log/cloudera-scm-alertpublisher
chmod 775 /var/lib/cloudera-scm-alertpublisher /var/log/cloudera-scm-alertpublisher
mkdir -vp /var/lib/cloudera-scm-headlamp /var/log/cloudera-scm-headlamp
chown cloudera-scm:cloudera-scm /var/lib/cloudera-scm-headlamp /var/log/cloudera-scm-headlamp
chown cloudera-scm:cloudera-scm /var/lib/cloudera-scm-headlamp /var/log/cloudera-scm-headlamp
chmod 775 /var/lib/cloudera-scm-headlamp /var/log/cloudera-scm-headlamp
mkdir -vp /var/lib/cloudera-service-monitor /var/log/cloudera-service-monitor
chown cloudera-scm:cloudera-scm /var/lib/cloudera-service-monitor /var/log/cloudera-service-monitor
chmod 775 /var/lib/cloudera-service-monitor /var/log/cloudera-service-monitor
mkdir -vp /var/lib/cloudera-scm-navigator /var/log/cloudera-scm-navigator
chown cloudera-scm:cloudera-scm /var/lib/cloudera-scm-navigator /var/log/cloudera-scm-navigator
chmod 775 /var/lib/cloudera-scm-navigator /var/log/cloudera-scm-navigator
mkdir -vp /var/lib/hadoop-hdfs /var/log/hadoop-hdfs
chown cloudera-scm:cloudera-scm /var/lib/hadoop-hdfs /var/log/hadoop-hdfs
chmod 775 /var/lib/hadoop-hdfs /var/log/hadoop-hdfs
mkdir -vp /dfs
chown cloudera-scm:cloudera-scm /dfs
chmod 775 /dfs
mkdir -vp /var/run/hdfs-sockets
chown cloudera-scm:cloudera-scm /var/run/hdfs-sockets
chmod 755 /var/run/hdfs-sockets/
#java.io.IOException: the path component: '/var/run/hdfs-sockets' is group-writable, and the group is not root. Its permissions are 0775, and it is owned by gid 981. Please fix this or select a different socket path.
#changed to 755
#chmod 755 hdfs-sockets
mkdir -vp /var/lib/solr /var/log/solr
chown cloudera-scm:cloudera-scm /var/lib/solr
chmod 775 /var/lib/solr
mkdir -vp /var/lib/hbase /var/log/hbase
chown cloudera-scm:cloudera-scm /var/lib/hbase /var/log/hbase
chown cloudera-scm:cloudera-scm /var/log/solr
chmod 775 /var/lib/solr /var/log/solr
chmod 775 /var/lib/hbase /var/log/hbase
mkdir -vp /var/log/hbase-solr /var/lib/hbase-solr
chown cloudera-scm:cloudera-scm /var/log/hbase-solr /var/lib/hbase-solr
chmod 775 /var/log/hbase-solr /var/lib/hbase-solr
mkdir -vp /var/lib/hadoop-yarn /var/log/hadoop-yarn /var/log/hadoop-mapreduce
chown cloudera-scm:cloudera-scm /var/lib/hadoop-yarn /var/log/hadoop-yarn /var/log/hadoop-mapreduce
chmod 775 /var/lib/hadoop-yarn /var/log/hadoop-yarn /var/log/hadoop-mapreduce
mkdir /yarn
chown cloudera-scm:cloudera-scm /yarn
chmod 755 /yarn
mkdir -vp /var/log/spark /var/log/hive
chown cloudera-scm:cloudera-scm /var/log/spark /var/log/hive
chmod 775 /var/log/spark /var/log/hive
mkdir -vp /var/lib/hadoop-httpfs /var/lib/oozie /var/lib/sqoop2 /var/lib/solr
chown cloudera-scm:cloudera-scm /var/lib/hadoop-httpfs /var/lib/oozie /var/lib/sqoop2 /var/lib/solr
chmod 775 /var/lib/hadoop-httpfs /var/lib/oozie /var/lib/sqoop2 /var/lib/solr
mkdir -vp /var/log/oozie /var/lib/oozie /var/log/catalogd /var/log/impalad /var/log/impala-minidumps /var/lib/impala /var/log/statestore
chown cloudera-scm:cloudera-scm /var/log/oozie /var/lib/oozie /var/log/catalogd /var/log/impalad /var/log/impala-minidumps /var/lib/impala /var/log/statestore
chmod 775 /var/log/oozie /var/lib/oozie /var/log/catalogd /var/log/impalad /var/log/impala-minidumps /var/lib/impala /var/log/statestore
mkdir /impala
chown cloudera-scm:cloudera-scm /impala
chmod 755 /impala
#Saw warning and did the below:
chmod 755 /var/lib/oozie
chmod 755 /var/log/oozie
mkdir /var/log/hue
chown cloudera-scm:cloudera-scm /var/log/hue
chmod 775 /var/log/hue
====== End ======
07] Once all the above folders created, Download CDH Installer:
wget https://archive.cloudera.com/cm5/installer/latest/cloudera-manager-installer.bin
08] chmod to execute:
chmod u+x cloudera-manager-installer.bin
09] Execute Installer:
./cloudera-manager-installer.bin
Follow the steps from the GUI. Once all done, you will see a notification stating that localhost:7180 will open in browser. Press the close button and you will see browser open's up with that link - dont panic if you see page not found or no connection error. Wait for couple of minutes and then refresh and continue the installation.
Un-Installing CDH:
In case if you want to wipe out (un-install) and start over CDH installation again then run the following to un-install and remove all the folders and start over from Step#6:
====== Begin ======
/usr/share/cmf/uninstall-cloudera-manager.sh
read -p "Enter to continue..."
service cloudera-scm-server stop
read -p "Enter to continue..."
service cloudera-scm-server-db stop
read -p "Enter to continue..."
yum remove cloudera-manager-server
read -p "Enter to continue..."
yum remove cloudera-manager-server-db-2
read -p "Enter to continue..."
service cloudera-scm-agent next_stop_hard
read -p "Enter to continue..."
service cloudera-scm-agent stop
read -p "Enter to continue..."
yum remove 'cloudera-manager-*'
read -p "Enter to continue..."
sudo yum remove 'cloudera-manager-*' avro-tools crunch flume-ng hadoop-hdfs-fuse hadoop-hdfs-nfs3 hadoop-httpfs hadoop-kms hbase-solr hive-hbase hive-webhcat hue-beeswax hue-hbase hue-impala hue-pig hue-plugins hue-rdbms hue-search hue-spark hue-sqoop hue-zookeeper impala impala-shell kite llama mahout oozie pig pig-udf-datafu search sentry solr-mapreduce spark-core spark-master spark-worker spark-history-server spark-python sqoop sqoop2 whirr hue-common oozie-client solr solr-doc sqoop2-client zookeeper
read -p "Enter to continue..."
yum clean all
read -p "Enter to continue..."
umount cm_processes
read -p "Enter to continue..."
rm -rf /usr/share/cmf /var/lib/cloudera* /var/cache/yum/cloudera* /var/log/cloudera* /var/run/cloudera*
read -p "Enter to continue..."
rm /tmp/.scm_prepare_node.lock
read -p "Enter to continue..."
rm -rf /var/lib/flume-ng /var/lib/hadoop* /var/lib/hue /var/lib/navigator /var/lib/oozie /var/lib/solr /var/lib/sqoop* /var/lib/zookeeper
read -p "Enter to continue..."
rm -rf /var/lib/zookeeper
rm -rf /var/log/zookeeper
rm -rf /var/lib/zookeeper/version-2
rm -rf /cloudera_manager_zookeeper_canary
rm -rf /var/lib/cloudera-scm-eventserver
rm -rf /var/log/cloudera-scm-eventserver directory
rm -rf /var/lib/cloudera-scm-firehose /var/log/cloudera-scm-firehose
rm -rf /var/lib/cloudera-scm-alertpublisher /var/log/cloudera-scm-alertpublisher
rm -rf /var/lib/cloudera-scm-headlamp /var/log/cloudera-scm-headlamp
rm -rf /var/lib/cloudera-service-monitor /var/log/cloudera-service-monitor
rm -rf /var/lib/cloudera-scm-navigator /var/log/cloudera-scm-navigator
rm -rf /var/lib/hadoop-hdfs /var/log/hadoop-hdfs
rm -rf /dfs
rm -rf /var/run/hdfs-sockets
rm -rf /var/lib/solr /var/log/solr
rm -rf /var/lib/hbase /var/log/hbase
rm -rf /var/log/hbase-solr /var/lib/hbase-solr
rm -rf /var/lib/hadoop-yarn /var/log/hadoop-yarn /var/log/hadoop-mapreduce
rm -rf /yarn
rm -rf /var/log/spark /var/log/hive
rm -rf /var/lib/hadoop-httpfs /var/lib/oozie /var/lib/sqoop2 /var/lib/solr
rm -rf /var/log/oozie /var/lib/oozie /var/log/catalogd /var/log/impalad /var/log/impala-minidumps /var/lib/impala /var/log/statestore
rm -rf /var/run/hdfs-sockets /var/log/hue
rm -rf /impala
rm -rf /usr/lib64/cmf
====== End ======
