# start hadoop cluster

### test configuration of hadoop
hadoop conftest [execute in master, worker and edge nodes]

### display hadoop version
hadoop version

### display hadoop environment variables
hadoop envvars

### format the namenode, create namenode directory/ datanode directory configured in 
	hdfs-site.xml
hdfs namenode -format

### start namenode and datanodes
start-dfs.sh [execute in master node]

### start resourcemanager and nodemanager
start-yarn.sh [execute in master node]

# execute the remaining commands in edge node

### report of hadoop cluster
hdfs dfsadmin -report

### information regarding master node
hdfs getconf -namenodes

### print classpath
hadoop classpath/ hdfs classpath

### make directory /user/hadoop/ in hdfs
hdfs dfs -mkdir -p /user/hadoop/
hdfs dfs -ls /user/hadoop/

### empty trash directory of hadoop
hdfs dfs -expunge

### after adding new node, we need to activate it without shuting down other nodes
hadoop dfsadmin refreshNodes (updated in *.xml file also)

### metasave provides more information than report
hdfs dfsadmin -metasave <filename>.txt (file saved in master node: /usr/local/hadoop/logs/)

### name quota - maximum number of files/directories
hdfs dfsadmin -setQuota N /user/raman/ 

### space quota - maximum files/directories space (for a file)
hdfs dfsadmin -setSpaceQuota 10g /user/raman/

### display quota statistics of the directory
hdfs dfs -count -q /user/raman/

### clear space quota
hdfs dfsadmin -clrSpaceQuota /user/raman

### create a file 
hdfs dfs -touchz /user/hadoop/report.txt

### delete a file
hdfs dfs -rm /user/hadoop/report.txt
hdfs dfs -rm -skipTrash /user/hadoop/report.txt

### change the file access privileges on the HDFS directory
hdfs dfs -chmod -R 755 /user

### check the health of the Hadoop file system
hdfs fsck /

### executing hdfs commands with sudo privileges by non-hadoop user
sudo -u hdfs hdfs fsck/
sudo -u hdfs hdfs dfsadmin -report

### creating hdfs/hadoop users [on edge node only]
sudo addgroup <groupname>
sudo adduser --ingroup <groupname> <username>


sudo addgroup analysts
sudo adduser --ingroup analysts kesavan
password: uiop[]\

$HADOOP_HOME/sbin/refresh-namenodes.sh

hdfs dfs -mkdir -p /tmp/hadoop-kesavan
hdfs dfs -chmod -R 777 /tmp/hadoop-kesavan

hdfs dfs -mkdir /user/kesavan

hdfs dfs -ls /user

hdfs dfs -chown -R kesavan:analysts /user/kesavan

hdfs dfsadmin -refreshUserToGroupsMappings

### copy the file from local file system to HDFS
hdfs dfs -put /home/raman/.bashrc /user/raman/
hdfs dfs -cp /user/raman/.bashrc /user/kesavan/.bashrc
hdfs dfs -chmod kesavan:analysts /user/kesavan/.bashrc

### ACL - Access Control Lists
ACLs are useful for implementing permissions requirements that differ from the natural 
organizational hierarchy of users and groups. An ACL provides a way to set different 
permissions for specific named users or named groups, not only the file owner and the file
group.

ACL shell commands

display the ACLs of files and directories:
hdfs dfs -getfacl /path
hdfs dfs -getfacl -R /path

set the ACLs of files and directories:
-m: Modify ACL. New entries are added to the ACL, and existing entries are retained
hdfs dfs -setfacl -m user:raman:rw- /file
-x: Remove specified ACL entries. Other ACL entries are retained
hdfs dfs -setfacl -x user:raman /file
-b: Remove all but the base ACL entries. The entries for user, group and others are 
	retained for compatibility with permission bits
hdfs dfs -setfacl -b /file
-k: Remove the default ACL
hdfs dfs -setfacl -k /directory
hdfs dfs -setfacl --set user::rw-,user:raman:rw-,group::r--,group:analyst:r--,other::r-- /user/kesavan/file
-R: Recursively
hdfs dfs -setfacl -R -m user:kesavan:r-x /directory
hdfs dfs -setfacl -m default:user:kesavan:r-x /directory

### display files
hdfs dfs -text <src>
