hadoopref.txt 3.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131
  1. # start hadoop cluster
  2. ### test configuration of hadoop
  3. hadoop conftest [execute in master, worker and edge nodes]
  4. ### display hadoop version
  5. hadoop version
  6. ### display hadoop environment variables
  7. hadoop envvars
  8. ### format the namenode, create namenode directory/ datanode directory configured in
  9. hdfs-site.xml
  10. hdfs namenode -format
  11. ### start namenode and datanodes
  12. start-dfs.sh [execute in master node]
  13. ### start resourcemanager and nodemanager
  14. start-yarn.sh [execute in master node]
  15. # execute the remaining commands in edge node
  16. ### report of hadoop cluster
  17. hdfs dfsadmin -report
  18. ### information regarding master node
  19. hdfs getconf -namenodes
  20. ### print classpath
  21. hadoop classpath/ hdfs classpath
  22. ### make directory /user/hadoop/ in hdfs
  23. hdfs dfs -mkdir -p /user/hadoop/
  24. hdfs dfs -ls /user/hadoop/
  25. ### empty trash directory of hadoop
  26. hdfs dfs -expunge
  27. ### after adding new node, we need to activate it without shuting down other nodes
  28. hadoop dfsadmin refreshNodes (updated in *.xml file also)
  29. ### metasave provides more information than report
  30. hdfs dfsadmin -metasave <filename>.txt (file saved in master node: /usr/local/hadoop/logs/)
  31. ### name quota - maximum number of files/directories
  32. hdfs dfsadmin -setQuota N /user/raman/
  33. ### space quota - maximum files/directories space (for a file)
  34. hdfs dfsadmin -setSpaceQuota 10g /user/raman/
  35. ### display quota statistics of the directory
  36. hdfs dfs -count -q /user/raman/
  37. ### clear space quota
  38. hdfs dfsadmin -clrSpaceQuota /user/raman
  39. ### create a file
  40. hdfs dfs -touchz /user/hadoop/report.txt
  41. ### delete a file
  42. hdfs dfs -rm /user/hadoop/report.txt
  43. hdfs dfs -rm -skipTrash /user/hadoop/report.txt
  44. ### change the file access privileges on the HDFS directory
  45. hdfs dfs -chmod -R 755 /user
  46. ### check the health of the Hadoop file system
  47. hdfs fsck /
  48. ### executing hdfs commands with sudo privileges by non-hadoop user
  49. sudo -u hdfs hdfs fsck/
  50. sudo -u hdfs hdfs dfsadmin -report
  51. ### creating hdfs/hadoop users [on edge node only]
  52. sudo addgroup <groupname>
  53. sudo adduser --ingroup <groupname> <username>
  54. sudo addgroup analysts
  55. sudo adduser --ingroup analysts kesavan
  56. password: uiop[]\
  57. $HADOOP_HOME/sbin/refresh-namenodes.sh
  58. hdfs dfs -mkdir -p /tmp/hadoop-kesavan
  59. hdfs dfs -chmod -R 777 /tmp/hadoop-kesavan
  60. hdfs dfs -mkdir /user/kesavan
  61. hdfs dfs -ls /user
  62. hdfs dfs -chown -R kesavan:analysts /user/kesavan
  63. hdfs dfsadmin -refreshUserToGroupsMappings
  64. ### copy the file from local file system to HDFS
  65. hdfs dfs -put /home/raman/.bashrc /user/raman/
  66. hdfs dfs -cp /user/raman/.bashrc /user/kesavan/.bashrc
  67. hdfs dfs -chmod kesavan:analysts /user/kesavan/.bashrc
  68. ### ACL - Access Control Lists
  69. ACLs are useful for implementing permissions requirements that differ from the natural
  70. organizational hierarchy of users and groups. An ACL provides a way to set different
  71. permissions for specific named users or named groups, not only the file owner and the file
  72. group.
  73. ACL shell commands
  74. display the ACLs of files and directories:
  75. hdfs dfs -getfacl /path
  76. hdfs dfs -getfacl -R /path
  77. set the ACLs of files and directories:
  78. -m: Modify ACL. New entries are added to the ACL, and existing entries are retained
  79. hdfs dfs -setfacl -m user:raman:rw- /file
  80. -x: Remove specified ACL entries. Other ACL entries are retained
  81. hdfs dfs -setfacl -x user:raman /file
  82. -b: Remove all but the base ACL entries. The entries for user, group and others are
  83. retained for compatibility with permission bits
  84. hdfs dfs -setfacl -b /file
  85. -k: Remove the default ACL
  86. hdfs dfs -setfacl -k /directory
  87. hdfs dfs -setfacl --set user::rw-,user:raman:rw-,group::r--,group:analyst:r--,other::r-- /user/kesavan/file
  88. -R: Recursively
  89. hdfs dfs -setfacl -R -m user:kesavan:r-x /directory
  90. hdfs dfs -setfacl -m default:user:kesavan:r-x /directory
  91. ### display files
  92. hdfs dfs -text <src>