1. HDFS 文件系统操作

1.1 基本文件操作

# 查看HDFS根目录
hdfs dfs -ls /

# 创建目录
hdfs dfs -mkdir /user
hdfs dfs -mkdir /user/hadoop
hdfs dfs -mkdir -p /data/input  # 递归创建目录

# 查看目录内容
hdfs dfs -ls /
hdfs dfs -ls -R /  # 递归查看
hdfs dfs -ls -h /  # 人类可读格式

1.2 文件上传和下载

# 上传本地文件到HDFS
hdfs dfs -put /home/hadoop/localfile.txt /user/hadoop/
hdfs dfs -copyFromLocal localfile.txt /user/hadoop/  # 同上

# 上传整个目录
hdfs dfs -put /home/hadoop/localdir /user/hadoop/

# 从HDFS下载文件
hdfs dfs -get /user/hadoop/hdfsfile.txt /home/hadoop/
hdfs dfs -copyToLocal /user/hadoop/hdfsfile.txt /home/hadoop/  # 同上

# 查看文件内容
hdfs dfs -cat /user/hadoop/file.txt
hdfs dfs -tail /user/hadoop/file.txt
hdfs dfs -head /user/hadoop/file.txt

1.3 文件管理和权限

# 复制文件
hdfs dfs -cp /user/hadoop/file1.txt /user/hadoop/file2.txt

# 移动/重命名文件
hdfs dfs -mv /user/hadoop/oldname.txt /user/hadoop/newname.txt

# 删除文件
hdfs dfs -rm /user/hadoop/file.txt
hdfs dfs -rm -r /user/hadoop/directory  # 递归删除目录

# 设置文件权限
hdfs dfs -chmod 755 /user/hadoop/file.txt
hdfs dfs -chown hadoop:hadoop /user/hadoop/file.txt
hdfs dfs -chgrp hadoop /user/hadoop/file.txt

# 查看文件大小
hdfs dfs -du -h /user/hadoop/  # 查看目录大小
hdfs dfs -df -h  # 查看HDFS磁盘使用情况

2. MapReduce 作业操作

2.1 运行内置示例

# 准备测试数据
echo "hello world hello hadoop hadoop spark" > input.txt
hdfs dfs -mkdir -p /input
hdfs dfs -put input.txt /input/

# 运行WordCount示例
hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar wordcount /input /output

# 查看结果
hdfs dfs -cat /output/part-r-00000

2.2 更多MapReduce示例

# 计算Pi值
hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar pi 10 100

# 排序示例
hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar randomwriter /random-data
hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar randomsample /random-data /sample-data 0.1

# Grep搜索示例
hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar grep /input /grep-output 'hello'

3. YARN 资源管理操作

3.1 查看集群状态

# 查看YARN节点状态
yarn node -list

# 查看运行中的应用
yarn application -list

# 查看应用详情
yarn application -status <application_id>

# 杀死应用
yarn application -kill <application_id>

3.2 资源管理

# 查看队列信息
yarn queue -status default

# 查看集群指标
yarn top

# 查看节点资源使用
yarn node -status <node_id>

4. 集群监控和管理

4.1 集群状态检查

# 检查HDFS健康状态
hdfs dfsadmin -report

# 检查HDFS安全模式
hdfs dfsadmin -safemode get
hdfs dfsadmin -safemode leave  # 退出安全模式

# 检查数据块信息
hdfs fsck / -files -blocks -locations

# 平衡数据分布
hdfs balancer -threshold 10

4.2 进程管理

# 查看所有Hadoop相关进程
jps

# 查看特定节点的进程
ssh hadoop-slave1 jps
ssh hadoop-slave2 jps

# 重启单个服务
hdfs --daemon stop datanode
hdfs --daemon start datanode

5. 实际案例操作

5.1 网站日志分析案例

# 1. 创建日志数据
cat > weblog.txt << EOF
192.168.1.1 - - [10/Oct/2023:10:30:00] "GET /index.html HTTP/1.1" 200 1024
192.168.1.2 - - [10/Oct/2023:10:31:00] "GET /about.html HTTP/1.1" 200 2048
192.168.1.1 - - [10/Oct/2023:10:32:00] "POST /login HTTP/1.1" 302 512
192.168.1.3 - - [10/Oct/2023:10:33:00] "GET /contact.html HTTP/1.1" 404 1024
EOF

# 2. 上传到HDFS
hdfs dfs -mkdir -p /logs/input
hdfs dfs -put weblog.txt /logs/input/

# 3. 创建分析程序（Python示例）
cat > log_analyzer.py << 'EOF'
#!/usr/bin/env python3
import sys

for line in sys.stdin:
    parts = line.split()
    if len(parts) >= 7:
        ip = parts[0]
        status = parts[8] if len(parts) > 8 else "200"
        print(f"{ip}\t1")
        print(f"status_{status}\t1")
EOF

# 4. 使用Hadoop Streaming运行（需要先安装hadoop-streaming）
hadoop jar $HADOOP_HOME/share/hadoop/tools/lib/hadoop-streaming-*.jar \
    -input /logs/input \
    -output /logs/output \
    -mapper log_analyzer.py \
    -reducer /bin/wc \
    -file log_analyzer.py

5.2 数据ETL处理案例

# 创建示例数据
cat > sales_data.csv << EOF
2023-10-01,ProductA,100,500.00
2023-10-01,ProductB,50,250.00
2023-10-02,ProductA,150,750.00
2023-10-02,ProductC,75,375.00
EOF

# 上传数据
hdfs dfs -mkdir -p /sales/input
hdfs dfs -put sales_data.csv /sales/input/

# 运行销售分析（使用Java MapReduce）
# 这里可以使用自定义的MapReduce程序进行分析

6. 性能测试和基准测试

6.1 使用TestDFSIO进行HDFS性能测试

# 写入测试（1GB数据，4个文件）
hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar TestDFSIO -write -nrFiles 4 -size 1GB

# 读取测试
hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar TestDFSIO -read -nrFiles 4 -size 1GB

# 清理测试数据
hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar TestDFSIO -clean

6.2 使用Terasort进行排序性能测试

# 生成测试数据
hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar teragen 10000000 /terasort-input

# 排序测试
hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar terasort /terasort-input /terasort-output

# 验证排序结果
hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar teravalidate /terasort-output /terasort-validate

7. 日常维护操作

7.1 备份和恢复

# 创建HDFS快照（需要先启用快照功能）
hdfs dfsadmin -allowSnapshot /important-data
hdfs dfs -createSnapshot /important-data backup-20231010

# 从快照恢复
hdfs dfs -cp /important-data/.snapshot/backup-20231010/file.txt /important-data/restored-file.txt

7.2 日志分析

# 查看NameNode日志
tail -f /opt/hadoop/logs/hadoop-hadoop-namenode-hadoop-master.log

# 查看DataNode日志
tail -f /opt/hadoop/logs/hadoop-hadoop-datanode-hadoop-slave1.log

# 查看YARN日志
tail -f /opt/hadoop/logs/yarn-hadoop-resourcemanager-hadoop-master.log

8. 实用脚本和工具

8.1 创建集群监控脚本

cat > cluster_monitor.sh << 'EOF'
#!/bin/bash

echo "=== Hadoop集群监控报告 ==="
echo "生成时间: $(date)"
echo ""

echo "=== HDFS状态 ==="
hdfs dfsadmin -report | grep -E "Configured Capacity|Present Capacity|DFS Used|DFS Remaining"

echo ""
echo "=== 数据节点状态 ==="
hdfs dfsadmin -report | grep "Live datanodes"

echo ""
echo "=== YARN节点状态 ==="
yarn node -list | grep "Total nodes"

echo ""
echo "=== 运行中的应用 ==="
yarn application -list | grep -v "Total" | grep "RUNNING"

echo ""
echo "=== 磁盘使用情况 ==="
hdfs dfs -df -h

echo ""
echo "=== 集群进程状态 ==="
echo "主节点:"
ssh hadoop-master jps | grep -v Jps
echo ""
echo "从节点1:"
ssh hadoop-slave1 jps | grep -v Jps
echo ""
echo "从节点2:"
ssh hadoop-slave2 jps | grep -v Jps
EOF

chmod +x cluster_monitor.sh
./cluster_monitor.sh

8.2 批量操作脚本

cat > batch_hdfs_ops.sh << 'EOF'
#!/bin/bash

# 批量创建标准目录结构
directories=("/data/raw" "/data/processed" "/data/backup" "/logs" "/tmp")

for dir in "${directories[@]}"; do
    echo "创建目录: $dir"
    hdfs dfs -mkdir -p $dir
    hdfs dfs -chmod 755 $dir
done

# 设置配额（限制目录大小）
hdfs dfsadmin -setSpaceQuota 10G /data/raw
hdfs dfsadmin -setSpaceQuota 20G /data/processed

echo "目录结构创建完成"
hdfs dfs -ls -R /
EOF