关于linux学习中awk的基本知识参考： linux文本编辑之awk
awk分析拟南芥gff文件
下载拟南芥gff文件
curl -O ftp://ftp.arabidopsis.org/home/tair/Genes/TAIR10_genome_release/TAIR10_gff3/TAIR10_GFF3_genes.gff
我们可以先来查看一下gff格式是什么样子的
less -S TAIR10_GFF3_genes.gff |head
Chr1 TAIR10 chromosome 1 30427671 . . . ID=Chr1;Name=Chr1
Chr1 TAIR10 gene 3631 5899 . + . ID=AT1G01010;Note=protein_coding_gene;Name=AT1G01010
Chr1 TAIR10 mRNA 3631 5899 . + . ID=AT1G01010.1;Parent=AT1G01010;Name=AT1G01010.1;Index=1
Chr1 TAIR10 protein 3760 5630 . + . ID=AT1G01010.1-Protein;Name=AT1G01010.1;Derives_from=AT1G01010.1
Chr1 TAIR10 exon 3631 3913 . + . Parent=AT1G01010.1
Chr1 TAIR10 five_prime_UTR 3631 3759 . + . Parent=AT1G01010.1
Chr1 TAIR10 CDS 3760 3913 . + 0 Parent=AT1G01010.1,AT1G01010.1-Protein;
Chr1 TAIR10 exon 3996 4276 . + . Parent=AT1G01010.1
Chr1 TAIR10 CDS 3996 4276 . + 2 Parent=AT1G01010.1,AT1G01010.1-Protein;
Chr1 TAIR10 exon 4486 4605 . + . Parent=AT1G01010.1
gff文件是tab分隔的文件
第1列是染色体信息
第2列是gff注释数据来源
第3列为特征（feature）即属于gene还是mRNA还是CDS等等
第4和5列分别是这个特征序列的起始和终止位置
第6列是得分，可以是序列相似性比对时的E-values值或者基因预测是的P-values值, ”.”表示为空
第7列是表示序列的方向：正义链为+，反义链为-
第8列仅为对CDS的注释，表示起始编码的位置，有效值为0、1、2
第9列为注释信息
$1, $2, $3指的就是第1-3列的数据
$ cat TAIR10_GFF3_genes.gff | awk ' { print $1, $2, $3 } ' | head -5
Chr1 TAIR10 chromosome
Chr1 TAIR10 gene
Chr1 TAIR10 mRNA
Chr1 TAIR10 protein
Chr1 TAIR10 exon
这（基本上）等同于截取列
$ cat TAIR10_GFF3_genes.gff | cut -f 1,2,3 | head -5
Chr1 TAIR10 chromosome
Chr1 TAIR10 gene
Chr1 TAIR10 mRNA
Chr1 TAIR10 protein
Chr1 TAIR10 exon
我们可以进行计算。每个特征序列长度是多少？
第5列的数值减去第4列的数值后+1，即得到特征序列的长度
$ cat TAIR10_GFF3_genes.gff | awk ' { print $3, $5-$4 + 1 } ' | head -8
chromosome 30427671
gene 2269
mRNA 2269
protein 1871
exon 283
five_prime_UTR 129
CDS 154
exon 281
我们可以利用模式匹配来提取CDS特征
cat TAIR10_GFF3_genes.gff | awk '$3 =="gene" { print $3, $5-$4 + 1, $9 } '| head -5
cat TAIR10_GFF3_genes.gff | awk '{if($3 =="gene") { print $3, $5-$4 + 1, $9 }} '| head -5
gene 2269 ID=AT1G01010;Note=protein_coding_gene;Name=AT1G01010
gene 2810 ID=AT1G01020;Note=protein_coding_gene;Name=AT1G01020
gene 2066 ID=AT1G01030;Note=protein_coding_gene;Name=AT1G01030
gene 8082 ID=AT1G01040;Note=protein_coding_gene;Name=AT1G01040
gene 207 ID=AT1G01046;Note=miRNA;Name=AT1G01046
计算所有基因的累积长度
cat TAIR10_GFF3_genes.gff | awk '$3 =="gene" { len=$5-$4 + 1; size += len; print "Size:", size } '
.....................
Size: 61219702
Size: 61220038
Size: 61220156
Size: 61222091
Size: 61222409
Size: 61223024
计算所有CDS的累积长度
cat TAIR10_GFF3_genes.gff | awk '$3 =="CDS" { len=$5-$4 + 1; size += len; print "Size:", size } '
............................
Size: 43543450
Size: 43543888
Size: 43545472
Size: 43545808
Size: 43546126
Size: 43546741
119667750是拟南芥（Col-0）基因组的大小
可以用下边的代码自行计算
$ cat TAIR10_GFF3_genes.gff |awk '$3 == "chromosome"{len=$5-$4 + 1; size += len; print "Size:", size,$9 } '
Size: 30427671 ID=Chr1;Name=Chr1
Size: 50125960 ID=Chr2;Name=Chr2
Size: 73585790 ID=Chr3;Name=Chr3
Size: 92170846 ID=Chr4;Name=Chr4
Size: 119146348 ID=Chr5;Name=Chr5
Size: 119300826 ID=ChrC;Name=ChrC
Size: 119667750 ID=ChrM;Name=ChrM
根据特征（features）把文件分开
cat NC.gff | awk ' $3=="gene" { print $0 }' >> NC-genes.gff
less -S NC-genes.gff|head -5
##gff-version 2
NC_002549 - gene 56 3026 . + . gene "NP" ; locus_tag "ZEBOVgp1" ; db_xref "GeneID:911830"
NC_002549 - gene 3032 4407 . + . gene "VP35" ; locus_tag "ZEBOVgp2" ; db_xref "GeneID:911827"
NC_002549 - gene 4390 5894 . + . gene "VP40" ; locus_tag "ZEBOVgp3" ; db_xref "GeneID:911825"
NC_002549 - gene 5900 8305 . + . gene "GP" ; locus_tag "ZEBOVgp4" ; db_xref "GeneID:911829"
cat NC.gff | awk ' $3=="CDS" { print $0 }' >> NC-cds.gff
less -S NC-cds.gff|head -5
提取启动子区域
cat TAIR10_GFF3_genes.gff| awk 'BEGIN{OFS=FS="\t"}{if($3=="gene"){if($7=="+"){start=$4-1;up=start-1000;if(up<0) up=0;dw=start+500;print $1,up,dw,$7;} else if($7=="-"){start=$5-1; up=start+1000; dw=start-500; if(dw<0) dw=0; print $1,dw,up,$7}}}' >TAIR10.promoter.bed
针对特定列的计算，比如wig文件的标准化
$ cat ehbio.wig
variableStep chrom=chr2
300701 12.5
300702 12.5
300703 12.5
300704 12.5
300705 12.0
$ awk 'BEGIN{OFS=FS="\t"}{$2=$2*10^6/(2.5*10^6); print $0}' ehbio.wig
variableStep chrom=chr2 0
300701 5
300702 5
300703 5
300704 5
300705 4.8计算某列内容出现的次数
$ cat count
ID Type
Pou5f1 Pluripotency
Nanog Pluripotency
Sox2 Neuron
Tet1 Epigenetic
Tet3 Epigenetic
Myc Oncogene
$ awk 'BEGIN{OFS=FS="\t"}{if(FNR>1) a[$2]+=1;}END{print "Count\tType"; for(i in a) print a[i],i;}' count
Count Type
2 Pluripotency
1 Oncogene
1 Neuron
2 Epigenetic数据矩阵的格式化输出
cat numertic.matrix
ID A B C
a 1.002 1.234 1.999
b 2.333 4.232 0.889
$ awk '{if(FNR==1) print $0;else {printf "%s%s",$1,FS;for (i=2;i<=NF;i++) printf "%.1f%s",$i, (i==NF?RS:FS)}}' numertic.matrix
ID A B C
a 1.0 1.2 2.0
b 2.3 4.2 0.9判断FASTQ文件中，输出质量值的长度是与序列长度不一致的序列ID
zcat Test_2.fq.gz | awk '{if(FNR%4==1) ID=$0; else if(FNR%4==2) seq_len=length($0); else if(FNR%4==0) {quality_len=length($0); if(seq_len!=quality_len) print ID; }}'筛选差异基因
cat de_gene
ID log2fc padj
A 1 0.001
B -1 0.001
C 1 0.001
D 2 0.0001
E -0.51 0.051
F 0.1 0.1
G 1 0.1
awk '$3<0.05 || NR==1' de_gene
ID log2fc padj
A 1 0.001
B -1 0.001
C 1 0.001
D 2 0.0001
awk 'BEGIN{OFS=FS="\t"}{if(FNR==1) print $0;else{abs_log2fc=($2<0?$2*(-1):$2);if(abs_log2fc>=1 && $3<0.05) print $0;}}' de_gene
ID log2fc padj
A 1 0.001
B -1 0.001
C 1 0.001
D 2 0.0001
存储到不同的文件
awk 'BEGIN{OFS=FS="\t"; up="up"; dw="dw";}{if(FNR==1) {print $0 >up; print $0 >dw;} else if ($3<0.05) {if($2>=1) print $0>up; else if($2<=-1) print $0>dw}}' de_gene
$ head up dw
==> up <==
ID log2fc padj
A 1 0.001
C 1 0.001
D 2 0.0001
==> dw <==
ID log2fc padj
B -1 0.001ID map，常用于转换序列的ID、提取信息、合并信息等
$ cat id_map
ENSM Symbol Entrez
ENSG00000280516 TMEM42 693149
ENSG00000281886 TGM4 7047
ENSG00000280873 DGKD 8527
ENSG00000281244 ADAMTS13 11093
ENSG00000280701 RP11-272D20.2
ENSG00000280674 ZDHHC3 51304
ENSG00000281623 Y_RNA
ENSG00000280479 CACFD1 11094
ENSG00000281165 SLC2A6 11182
ENSG00000281879 ABO 28
ENSG00000282873 BCL7A 605
ENSG00000280651 AC156455.1 100506691
[sunchengquan 23:19:33 ~]
$ cat ensm
ENSG00000281244
ENSG00000281165
ENSG00000282873
$ awk 'BEGIN{OFS=FS="\t"} ARGIND==1{if(FNR>1) {map[$1]=$3};} ARGIND==2{print map[$1];}' id_map ensm
11093
11182
605
转换大小写, toupper, tolower
$ cat symbol
Tgm4
Dgkd
Abo
$ awk '{print toupper($1)}' symbol
TGM4
DGKD
ABO
$ awk 'BEGIN{OFS=FS="\t"}ARGIND==1{if(FNR>1) ensm2entrez[$2]=$3;}ARGIND==2{print ensm2entrez[toupper($1)];}' id_map symbol
7047
8527
28awk数值操作
# log2对数
awk 'BEGIN{OFS="\t";FS="\t"}{print log($0)/log(2)}' file
# 取整,四舍五入
awk 'BEGIN{OFS="\t";FS="\t"}{print int($1+0.5);}' file字符串匹配
$ cat ens.bed
1 100 105
2 100 105
3 100 105
Mt 100 105
X 100 105
[sunchengquan 16:11:25 ~]
$ awk 'BEGIN{OFS=FS="\t"}{if($1~/^[0-9XY]/) $1="chr"$1; else if($1~/M.*/)gsub(/M.*/, "chrM", $1); print $0}' ens.bed
chr1 100 105
chr2 100 105
chr3 100 105
chrM 100 105
chrX 100 105字符串分割
$ cat trinity_id
Trinity_C1_g1_i1
Trinity_C1_g1_i2
Trinity_C1_g1_i3
Trinity_C2_g1_i1
Trinity_C3_g1_i1
Trinity_C3_g3_i2
[sunchengquan 16:40:11 ~]
$ awk 'BEGIN{OFS=FS="\t"}{count=split($1, geneL, "_"); gene=geneL[1];print gene}' trinity_id
Trinity
Trinity
Trinity
Trinity
Trinity
Trinity
[sunchengquan 16:40:24 ~]
$ awk 'BEGIN{OFS=FS="\t"}{count=split($1, geneL, "_"); gene=geneL[1];print geneL[1],geneL[2];}' trinity_id
Trinity C1
Trinity C1
Trinity C1
Trinity C2
Trinity C3
Trinity C3
[sunchengquan 16:41:27 ~]
$ awk 'BEGIN{OFS=FS="\t"}{count=split($1, geneL, "_"); gene=geneL[1]; for(i=2;i<count;i++) gene=gene"_"geneL[i]; print gene,$1;}' trinity_id
Trinity_C1_g1 Trinity_C1_g1_i1
Trinity_C1_g1 Trinity_C1_g1_i2
Trinity_C1_g1 Trinity_C1_g1_i3
Trinity_C2_g1 Trinity_C2_g1_i1
Trinity_C3_g1 Trinity_C3_g1_i1
Trinity_C3_g3 Trinity_C3_g3_i2
[sunchengquan 16:47:16 ~]
$ awk 'BEGIN{OFS=FS="\t"}{count=split($1, geneL, "_"); gene=geneL[1]; for(i=2;i<=count;i++) gene=gene"_"geneL[i]; print gene,$1;}' trinity_id
Trinity_C1_g1_i1 Trinity_C1_g1_i1
Trinity_C1_g1_i2 Trinity_C1_g1_i2
Trinity_C1_g1_i3 Trinity_C1_g1_i3
Trinity_C2_g1_i1 Trinity_C2_g1_i1
Trinity_C3_g1_i1 Trinity_C3_g1_i1
Trinity_C3_g3_i2 Trinity_C3_g3_i2