Shell文本三剑客之awk

1. /etc/passwd 的所有内容显示
    awk '{print $0}' /etc/passwd
2. 指定分隔符分割后的一列，/etc/passwd
    awk 'BEGIN{FS=":"}{print $7}' /etc/passwd
3. 输出当前行某个文件分割后的数据的个数,NF(Number Field) 
    awk 'BEGIN{FS=":"}{print NF}' /etc/passwd
4. NR(Number Row),处理行号
    awk 'BEGIN{FS=":"}{print NR}' /etc/passwd
    当有多个文件时，直接加，如果是FNR(File Number Row)就会每个文件的行号从1开始
5. FS(Field Separator) 字段分隔符，不指定是以空格或tab键分割
    awk 'BEGIN{FS=":"}{print $7}' /etc/passwd
6. RS(Row Separator) 行分割符，默认换行符
    awk 'BEGIN{RS="---"}{print $0}' rs.txt
    # rs.txt 内容aaaaaaaaaaa---bbbbbbbbbbbbbbbb---cccccccccccccc
7. OFS(Output Field Separator) 字段输出分隔符，默认空格
    awk 'BEGIN{FS=":";OFS="##"}{print $1,$2}' /etc/passwd
    # 如果$1 $2不以逗号分割OFS不起作用
8. ORS(Output Row Separator) 输出行分隔符，默认换行
    awk 'BEGIN{ORS="-_+";FS=":"}{print $1,$2}' /etc/passwd
9. FILENAME 文件名
    awk '{print FILENAME}' /etc/passwd
    # 多少行数据，输出多少个
10. ARGC 变量个数
    awk '{print ARGC}' /etc/passwd
    # 2个参数， awk和/etc/passwd

格式符示例:
    1、以字符串格式打印/etc/passwd中的第7个字段，以":"作 为分隔符
        awk 'BEGIN{FS=":"}{printf "%s\n",$7}' /etc/passwd
    2、以10进制格式打印/etc/passwd中的第3个字段，以":"作 为分隔符
        awk 'BEGIN{FS=":"}{printf "%d\n",$3}' /etc/passwd
    3、以浮点数格式打印/etc/passwd中的第3个字段，以":"作为分隔符
        awk 'BEGIN{FS=":"}{printf "%f\n",$3}' /etc/passwd
    4、以16进制数格式打印/etc/passwd中的第3个字段，以":"作 为分隔符
        awk 'BEGIN{FS=":"}{printf "%x\n",$3}' /etc/passwd
    5、以8进制数格式打印/etc/passwd中的第3个字段，以":"作为分隔符
        awk 'BEGIN{FS=":"}{printf "%o\n",$3}' /etc/passwd
    6、以科学计数法格式打印/etc/passwd中的第3个字段，以":"作为分隔符
        awk 'BEGIN{FS=":"}{printf "%e\n",$3}' /etc/passwd
修饰符示例:
    1、左对齐格式
        awk 'BEGIN{FS=":"}{printf "%-20s %-20s\n",$1,$7}' /etc/passwd
    2、右对齐格式
        awk 'BEGIN{FS=":"}{printf "%20s %20s\n",$1,$7}' /etc/passwd
    3、打印8进制或16进制数字是在前面加#
        awk 'BEGIN{FS=":"}{printf "%#o\n",$3}' /etc/passwd
        awk 'BEGIN{FS=":"}{printf "%#x\n",$3}' /etc/passwd

1、使用awk计算/etc/services中的空白行数量)
    # 空白行的表示^$
    awk '/^$/{sum++}END{print sum}' /etc/services
2、计算学生课程分数平均值，学生课程文件内容如下:
    Allen   80  90  96  98
    Mike    93  98  92  91
    Zhang   78  76  87  92
    Jerry   86  89  68  92
    Han     85  95  75  90
    Li  78  88  98  100 
awk '{sum=$2+$3+$4+$5; AVG=sum/4; printf "%-8s%-8d%-8d%-8d%-8d%0.2f\n",$1,$2,$3,$4,$5,AVG}' stu.txt
    
3、使用awk输出/etc/passwd文件的行数。分两种方式显示行数，一种是正序如1.2.3.4..
    awk '{line++; printf "%d:%s\n", line,$0}' /etc/passwd

1、以:为分隔符，只打印/etc/passwd中第3个字段的数值在50-100范围内的行信息
    awk 'BEGIN{FS=":"}{if(50<$3 && $3<100) print $0}' /etc/passwd


2、计算下列每个同学的平均分数，并且只打印平均分数大于90的同学姓名和分数信息
    Allen   80  90  96  98
    Mike    93  98  92  91
    Zhang   78  76  87  92
    Jerry   86  89  68  92
    Han     85  95  75  90
    Li  78  88  98  100 
awk '{sum=$2+$3+$4+$5; AVG=sum/4; if(AVG>90)printf "%-8s%-8d%-8d%-8d%-8d%0.2f\n",$1,$2,$3,$4,$5,AVG}' stu.txt

# 三种循环
awk 'BEGIN {i = 1; while (i < 6) { print i; ++i } }'
awk 'BEGIN { for (i = 1; i <= 5; ++i) print i }'
awk 'BEGIN {i = 1; do { print i; ++i } while (i < 6) }'

1、以:为分隔符，返回/etc/passwd中每行中每个字段的长度
    awk 'BEGIN{
            FS=":"
        }{
            i=0;
            while(i<=NF){
              printf "%d", length($i);
              if(i!=NF)printf ":"; 
              i++; 
            }
            printf "\n"
            
        }' /etc/passwd
2、搜索字符串"I have a dream"中出现"ea"字符串的位置
    awk 'BEGIN{str="I have a dream"; print index(str,"ea")}'
3、将字符串"Hadoop is a bigdata Framawork"全部转换为小写
    awk 'BEGIN{str="Hadoop is a bigdata Framawork"; print tolower(str)}'
4、将字符串"Hadoop is a bigdata Framawork" 全部转换为大写
    awk 'BEGIN{str="Hadoop is a bigdata Framawork"; print toupper(str)}'
5、将字符串"Hadoop Kafka Spark Storm HDFS YARN Zookeeper"， 按照空格为分隔符
    # 从1开始
    awk 'BEGIN{str="Hadoop Kafka Spark Storm HDFS YARN Zookeeper"; split(str, arr, " "); print arr[1]}'
6、搜索字符串"Tranction 2345 Start:Select * from master"第一个数字出现的位置
    awk 'BEGIN{str="Tranction 2345 Start:Select * from master"; print match(str, /[0-9]+/)}'

7、截取字符串" transaction start"的子串，截取条件从第4个字符开始，截取5位
    awk 'BEGIN{str=" transaction start"; print substr(str, 4,5)}'
8、替换字符串"Tranction 243 start, Event ID:9002"中第一个匹配到的数字串为$符号
    awk 'BEGIN{str="Tranction 243 start, Event ID:9002"; sub(/[0-9]+/, "$", str); print str}'

# -v
[root@master ~]# num1=100
[root@master ~]# var="You"
[root@master ~]# awk -v num2=$num1 -v var1=$var 'BEGIN{print num2, var1}'
100 You

# 字符串有空格需要引号
[root@master ~]# var="Hello Shell"
[root@master ~]# awk -v num2=$num1 -v var1=$var 'BEGIN{print num2, var1}'
awk: fatal: cannot open file `BEGIN{print num2, var1}' for reading (No such file or directory)
[root@master ~]# awk -v num2=$num1 -v var1="$var" 'BEGIN{print num2, var1}'
100 Hello Shell

# -f awk后面的BEGIN放入test.awk后执行
[root@master datas]# awk -v num2=$num1 -v var1="$var" -f test.awk 
100 Hello Shell

# -F
awk -F : 'BEGIN{}{print $1}' /etc/passwd

# -V
[root@master ~]# awk -V
GNU Awk 4.0.2

1. 统计主机上所有的TCP连接状态数，按照每个TCP状态分类
    netstat -an | grep tcp | awk '{array[$6]++}END{for(a in array) print a, array[a]}'
2. 计算横向数据总和，计算纵向数据总和
Allen   80  90  96  98
Mike    93  98  92  91
Zhang   78  76  87  92
Jerry   86  89  68  92
Han     85  95  75  90
Li  78  88  98  100 
    awk 'BEGIN{printf "%-10s%-10s%-10s%-10s%-10s%-10s\n", "Name","Chinese","English","Math","Physical","Total" }{
        array["Chinese"]+=$2
        array["English"]+=$3
        array["Math"]+=$4
        array["Physical"]+=$5
        total = $2+$3+$4+$5
        printf "%-10s%-10d%-10d%-10d%-10d%-10d\n", $1,$2,$3,$4,$5,total
    }END{
        printf "%-10s", ""
        for(a in array) printf "%-10d",array[a]

        printf "\n"
    }' student.txt

array=("If" "we" "can" "only" "encounter" "each" "other" "rather" "than" "stay" "with" "each" "other,then" "I" "wish" "we" "had never encountered")

打印元素:         echo ${array[2] }       
打印元素个数:     echo ${#array[@] }           
打印元素长度:     echo ${#array[3] }           
给元素赋值:       array[3]="Li"       
删除元素:         unset array[2] ;unset array       
分片访问:         echo ${array[@] :1:3}       
元素内容替换:     ${array[@]/e/E}只替换第-一个e;${array[@]//e/E}替换所有的e      

数组的遍历：
for a in array
do 
    echo $a
done

root@ubuntu:~# array=("If" "we" "can" "only" "encounter" "each" "other" "rather" "than" "stay" "with" "each" "other,then" "I" "wish" "we" "had never encountered")
root@ubuntu:~# echo ${array[2]}
can
root@ubuntu:~# echo ${array[*]}
If we can only encounter each other rather than stay with each other,then I wish we had never encountered
root@ubuntu:~# echo ${#array[*]}
17
root@ubuntu:~# echo ${#array[@]}
17
root@ubuntu:~# echo ${array[@]}
If we can only encounter each other rather than stay with each other,then I wish we had never encountered
root@ubuntu:~# unset array[2]
root@ubuntu:~# echo ${array[@]}
If we only encounter each other rather than stay with each other,then I wish we had never encountered
root@ubuntu:~# echo ${array[@]:1:3}
we only encounter
root@ubuntu:~# echo ${array[@]/we/we and you}
If we and you only encounter each other rather than stay with each other,then I wish we and you had never encountered

#!/bin/absh
# 

function create_random(){
    min=$1
    max=$(($2-$min+1))
    num=$(date+%s%N)
    echo $(($num%$max+$min))
}
INDEX=1

while true
do
    for user in A B C D E F G H I J K L
    do
        COUNT=$RANDOM
        NUM1=`create_random 1 $COUNT`
        NUM2=`expr $COUNT - $NUM1`
        echo "`date '+%Y-%m-%d %H:%M:%S'` $INDEX Batches:$user INSERT $COUNT DATA INTO DATABASE:mydb table log, INSERT $NUM1 records successfully, failed $NUM2 records" >> 123.txt 
        INDEX=`expr $INDEX + 1`
    done
done

(1)、统计每个人员分别插入了多少条record进数据库
    awk 'BEGIN{
        FS=" "
    }{
        array[substr($4, 9, 1)]++
    }END{
        for(a in array) print a, array[a]
    }' 123.txt
(2)、统计每个人分别插入成功了多少record,失败了多少record
13 17
    awk 'BEGIN{
        FS=" "
    }{
        str=substr($4, 9, 1)
        success[str]+=$13
        fail[str]+=$17
    }END{
        for(a in success) print a, success[a],fail[a]
    }' 123.txt
(3)、 将例子1和例子2结合起来，一起输出，输出每个人分别插入多少数据，多少成功,多少条失败
    awk 'BEGIN{
        FS=" "
        printf "%-10s%-10s%-10s%-10s\n","Name","Number","Success","Failure"
    }{
        str=substr($4, 9, 1)
        array[str]++
        success[str]+=$13
        fail[str]+=$17
    }END{

        for(a in success){
            printf "%-10s%-10d%-10d%-10d\n",a, array[a],success[a],fail[a]
        } 
    }' 123.txt
(4)、在例子3的基础上，加上结尾，统计全部插入记录数，成功记录数，失败记录数
awk 'BEGIN{
        FS=" "
        printf "%-10s%-10s%-10s%-10s\n","Name","Number","Success","Failure"
    }{
        str=substr($4, 9, 1)
        array[str]++
        success[str]+=$13
        fail[str]+=$17
    }END{

        for(a in success){
            Total+=array[a]
            sTotal+=success[a]
            fTotal+=fail[a]
            printf "%-10s%-10d%-10d%-10d\n",a, array[a],success[a],fail[a]
        } 
        printf "%-10s%-10d%-10d%-10d\n","TOTAL", Total,sTotal,fTotal
    }' 123.txt
(5)、查找丢失数据的现象，也就是成功+失败的记录数，不等于一共插入的记录数。找出这些数据并显示行号和对应日志信息
    修改123.txt文件的数据进行测试
    awk '{if($6!=$13+$17) printf "%d:%s\n", NR,$0 }' 123.txt