123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131 |
- # start hadoop cluster
- ### test configuration of hadoop
- hadoop conftest [execute in master, worker and edge nodes]
- ### display hadoop version
- hadoop version
- ### display hadoop environment variables
- hadoop envvars
- ### format the namenode, create namenode directory/ datanode directory configured in
- hdfs-site.xml
- hdfs namenode -format
- ### start namenode and datanodes
- start-dfs.sh [execute in master node]
- ### start resourcemanager and nodemanager
- start-yarn.sh [execute in master node]
- # execute the remaining commands in edge node
- ### report of hadoop cluster
- hdfs dfsadmin -report
- ### information regarding master node
- hdfs getconf -namenodes
- ### print classpath
- hadoop classpath/ hdfs classpath
- ### make directory /user/hadoop/ in hdfs
- hdfs dfs -mkdir -p /user/hadoop/
- hdfs dfs -ls /user/hadoop/
- ### empty trash directory of hadoop
- hdfs dfs -expunge
- ### after adding new node, we need to activate it without shuting down other nodes
- hadoop dfsadmin refreshNodes (updated in *.xml file also)
- ### metasave provides more information than report
- hdfs dfsadmin -metasave <filename>.txt (file saved in master node: /usr/local/hadoop/logs/)
- ### name quota - maximum number of files/directories
- hdfs dfsadmin -setQuota N /user/raman/
- ### space quota - maximum files/directories space (for a file)
- hdfs dfsadmin -setSpaceQuota 10g /user/raman/
- ### display quota statistics of the directory
- hdfs dfs -count -q /user/raman/
- ### clear space quota
- hdfs dfsadmin -clrSpaceQuota /user/raman
- ### create a file
- hdfs dfs -touchz /user/hadoop/report.txt
- ### delete a file
- hdfs dfs -rm /user/hadoop/report.txt
- hdfs dfs -rm -skipTrash /user/hadoop/report.txt
- ### change the file access privileges on the HDFS directory
- hdfs dfs -chmod -R 755 /user
- ### check the health of the Hadoop file system
- hdfs fsck /
- ### executing hdfs commands with sudo privileges by non-hadoop user
- sudo -u hdfs hdfs fsck/
- sudo -u hdfs hdfs dfsadmin -report
- ### creating hdfs/hadoop users [on edge node only]
- sudo addgroup <groupname>
- sudo adduser --ingroup <groupname> <username>
- sudo addgroup analysts
- sudo adduser --ingroup analysts kesavan
- password: uiop[]\
- $HADOOP_HOME/sbin/refresh-namenodes.sh
- hdfs dfs -mkdir -p /tmp/hadoop-kesavan
- hdfs dfs -chmod -R 777 /tmp/hadoop-kesavan
- hdfs dfs -mkdir /user/kesavan
- hdfs dfs -ls /user
- hdfs dfs -chown -R kesavan:analysts /user/kesavan
- hdfs dfsadmin -refreshUserToGroupsMappings
- ### copy the file from local file system to HDFS
- hdfs dfs -put /home/raman/.bashrc /user/raman/
- hdfs dfs -cp /user/raman/.bashrc /user/kesavan/.bashrc
- hdfs dfs -chmod kesavan:analysts /user/kesavan/.bashrc
- ### ACL - Access Control Lists
- ACLs are useful for implementing permissions requirements that differ from the natural
- organizational hierarchy of users and groups. An ACL provides a way to set different
- permissions for specific named users or named groups, not only the file owner and the file
- group.
- ACL shell commands
- display the ACLs of files and directories:
- hdfs dfs -getfacl /path
- hdfs dfs -getfacl -R /path
- set the ACLs of files and directories:
- -m: Modify ACL. New entries are added to the ACL, and existing entries are retained
- hdfs dfs -setfacl -m user:raman:rw- /file
- -x: Remove specified ACL entries. Other ACL entries are retained
- hdfs dfs -setfacl -x user:raman /file
- -b: Remove all but the base ACL entries. The entries for user, group and others are
- retained for compatibility with permission bits
- hdfs dfs -setfacl -b /file
- -k: Remove the default ACL
- hdfs dfs -setfacl -k /directory
- hdfs dfs -setfacl --set user::rw-,user:raman:rw-,group::r--,group:analyst:r--,other::r-- /user/kesavan/file
- -R: Recursively
- hdfs dfs -setfacl -R -m user:kesavan:r-x /directory
- hdfs dfs -setfacl -m default:user:kesavan:r-x /directory
- ### display files
- hdfs dfs -text <src>
|