七牛日志分析

七牛日志格式

116.231.10.133 HIT 0 [07/May/2019:09:04:56 +0800] "GET http://wdl1.cache.wps.cn/per-plugin/dl/addons/pool/win-i386/wpsminisite_3.0.0.37.7z HTTP/1.1" 206 66372 "-" "-"

Qt的相关代码

int getDownloadSize(QString line) {
    int idx = line.indexOf("HTTP/1.1\"");
    if(idx > 0) {
        QString sub = line.mid(idx);
        QStringList sets = sub.split(" ");
        int size = sets.at(2).toInt();
        return size;
    }
    return 0;
}
 
 
int main(int argc, char *argv[])
{
    QCoreApplication a(argc, argv);
 
    QString dpath("F:\\日志0508\\18");
 
    QDir d(dpath);
    QStringList fs = d.entryList();
    qint64 plug_total = 1;
    qint64 other_total = 1;
    for(int i = 0; i < fs.size(); i++) {
        QString f = fs.at(i);
        QFile file(dpath+"\\"+f);
        if(!file.open(QIODevice::ReadOnly|QIODevice::Text)) {
            continue;
        }
        int linecnt = 0;
        while(!file.atEnd()) {
            QByteArray line = file.readLine();
            if(line.indexOf("wdl1.cache.wps.cn") < 0) {
                continue;
            }
            int idx = line.indexOf("win-i386");
            int size = getDownloadSize(line);
            qint64 skidx = file.pos();
            if(idx > 0) {
                plug_total += size;
            }else{
                other_total += size;
            }
            linecnt++;
            if(linecnt % 10000 == 0) {
                printf("\r\nlinecnt:%d - skidx:%lld - plug_total:%lld - other_total:%lld - ratio:%5f", linecnt, skidx, plug_total, other_total, double(plug_total) / double(other_total));
            }
        }
        printf("\r\nlinecnt:%d - plug_total:%lld - other_total:%lld - ratio:%5f", linecnt, plug_total, other_total, double(plug_total) / double(other_total));
    }
    printf("\r\nplug_total:%lld - other_total:%lld - ratio:%5f", plug_total, other_total, double(plug_total) / double(other_total));
    return a.exec();
}

获取响应内容

log_format cntr escape=json '{"time":"$time_local",'
                        '"@source":"$server_addr",'
                        '"hostname":"$hostname",'
                        '"xforward":"$http_x_forwarded_for",'
                        '"remoteaddr":"$remote_addr",'
                        '"method":"$request_method",'
                        '"scheme":"$scheme",'
                        '"domain":"$server_name",'
                        '"referer":"$http_referer",'
                        '"requrl":"$request_uri",'
                        '"args":"$args",'
                        '"requestbody":"$request_body",'
                        '"contentlength":"$content_length",'
                        '"bodybytessend":$body_bytes_sent,'
                        '"status":$status,'
                        '"requesttime":$request_time,'
                        '"upstreamtime":"$upstream_response_time",'
                        '"upstreamaddr":"$upstream_addr",'
                        '"respbody":"$resp_body",'
                        '"respdown":"$resp_ctrl_down",'
                        '"useragent":"$http_user_agent"'
                        '}';
location /abc {
        #access_log off;
        access_log /data/logs/nginx/abc/access.log cntr buffer=32k flush=5s;
        error_log /data/logs/nginx/abc/error.log info;
 
        set $resp_ctrl_down -1;
        set $resp_body "";
        # only elk collection machine, can add the below lua code.
        body_filter_by_lua_block {
            local chunk, eof = ngx.arg[1], ngx.arg[2]
            ngx.var.resp_body=ngx.var.resp_body..chunk
            if eof then
               if string.find(ngx.var.resp_body,'"count":true') then
                  ngx.var.resp_ctrl_down = 1
               else
                  ngx.var.resp_ctrl_down = 0
               end
            end
        }
 
        proxy_pass http://abc/flow;
    }

fedora换阿里云镜像源

su -
cd /etc/yum.repos.d/
mv fedora.repo fedora.repo.backup
mv fedora-updates.repo fedora-updates.repo.backup
wget -O /etc/yum.repos.d/fedora.repo http://mirrors.aliyun.com/repo/fedora.repo
wget -O /etc/yum.repos.d/fedora-updates.repo http://mirrors.aliyun.com/repo/fedora-updates.repo
dnf clean all
dnf makecache

终端测试验证工具

http://invisible-island.net/xterm/ctlseqs/ctlseqs.html
http://www.xfree86.org/current/ctlseqs.html
Linux环境下,主要都是泛VT102的终端,以下是vt102终端。
https://vt100.net/docs/vt102-ug/introduction.html
https://vt100.net/docs/vt102-ug/contents.html

VT100,VT101,
参考开源:https://konsole.kde.org/

自动拉起服务脚本

crontab -e
*/1 * * * * sh /data/scripts/run-flow.sh start
#!/bin/sh
 
path_current=`pwd`
path_script=$(cd "$(dirname "$0")"; pwd)
mode=$1
 
logfile=/data/scripts/check.log
app_process=`ps -ef | grep "flowservice"| grep -v grep`
echo `date` >> $logfile
echo "ready to check...." >> $logfile
case "$mode" in
   'start')
        echo "$app_process" >> $logfile
        echo "it's ready to start op...."
        if test -n "$app_process"; then
                echo ""
                echo "$app_process"
                echo ""
        else
                cd $path_script   #进入脚本所在目录下,目的是使springboot的config目录生效。
                nohup /data/code/service.flow.wps.cn/flowservice --config=/data/code/service.flow.wps.cn/config/config-prod.toml > /data/code/service.flow.wps.cn/info.txt 2>&1 &
                echo "success to restart flowservice" >> $logfile
                cd $path_current
 
        fi
 
        echo 'success to start.'
        ;;
   'stop')
        echo "it's ready to check process..."
        if test -n "$app_process"; then
                echo "had find app process informaton"
                echo $app_process | awk '{print ($2)}' | xargs kill -9
        fi
        echo 'success to kill.'
        ;;
    *)
        basename=`basename "$0"`
        echo "Usage: $basename  {start|stop}  [ server options ]"
        exit 1
        ;;
esac
exit 1

GoLang的AST语法树

package main
 
import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
)
 
func main() {
	// src is the input for which we want to print the AST.
	src := `
// @router /login [post]
// @router2 /login2 [post]
package main
func main() {
	println("Hello, World!")
}
`
 
	// Create the AST by parsing src.
	fset := token.NewFileSet() // positions are relative to fset
	f, err := parser.ParseFile(fset, "", src, parser.ParseComments)
	if err != nil {
		panic(err)
	}
	for _, comment := range f.Comments {
		fmt.Println(comment.Text())
	}
 
	// Print the AST.
	ast.Print(fset, f)
 
}

运行结果如下:

@router /login [post]
@router2 /login2 [post]
 
     0  *ast.File {
     1  .  Doc: *ast.CommentGroup {
     2  .  .  List: []*ast.Comment (len = 2) {
     3  .  .  .  0: *ast.Comment {
     4  .  .  .  .  Slash: 2:1
     5  .  .  .  .  Text: "// @router /login [post]"
     6  .  .  .  }
     7  .  .  .  1: *ast.Comment {
     8  .  .  .  .  Slash: 3:1
     9  .  .  .  .  Text: "// @router2 /login2 [post]"
    10  .  .  .  }
    11  .  .  }
    12  .  }
    13  .  Package: 4:1
    14  .  Name: *ast.Ident {
    15  .  .  NamePos: 4:9
    16  .  .  Name: "main"
    17  .  }
    18  .  Decls: []ast.Decl (len = 1) {
    19  .  .  0: *ast.FuncDecl {
    20  .  .  .  Name: *ast.Ident {
    21  .  .  .  .  NamePos: 5:6
    22  .  .  .  .  Name: "main"
    23  .  .  .  .  Obj: *ast.Object {
    24  .  .  .  .  .  Kind: func
    25  .  .  .  .  .  Name: "main"
    26  .  .  .  .  .  Decl: *(obj @ 19)
    27  .  .  .  .  }
    28  .  .  .  }
    29  .  .  .  Type: *ast.FuncType {
    30  .  .  .  .  Func: 5:1
    31  .  .  .  .  Params: *ast.FieldList {
    32  .  .  .  .  .  Opening: 5:10
    33  .  .  .  .  .  Closing: 5:11
    34  .  .  .  .  }
    35  .  .  .  }
    36  .  .  .  Body: *ast.BlockStmt {
    37  .  .  .  .  Lbrace: 5:13
    38  .  .  .  .  List: []ast.Stmt (len = 1) {
    39  .  .  .  .  .  0: *ast.ExprStmt {
    40  .  .  .  .  .  .  X: *ast.CallExpr {
    41  .  .  .  .  .  .  .  Fun: *ast.Ident {
    42  .  .  .  .  .  .  .  .  NamePos: 6:2
    43  .  .  .  .  .  .  .  .  Name: "println"
    44  .  .  .  .  .  .  .  }
    45  .  .  .  .  .  .  .  Lparen: 6:9
    46  .  .  .  .  .  .  .  Args: []ast.Expr (len = 1) {
    47  .  .  .  .  .  .  .  .  0: *ast.BasicLit {
    48  .  .  .  .  .  .  .  .  .  ValuePos: 6:10
    49  .  .  .  .  .  .  .  .  .  Kind: STRING
    50  .  .  .  .  .  .  .  .  .  Value: "\"Hello, World!\""
    51  .  .  .  .  .  .  .  .  }
    52  .  .  .  .  .  .  .  }
    53  .  .  .  .  .  .  .  Ellipsis: -
    54  .  .  .  .  .  .  .  Rparen: 6:25
    55  .  .  .  .  .  .  }
    56  .  .  .  .  .  }
    57  .  .  .  .  }
    58  .  .  .  .  Rbrace: 7:1
    59  .  .  .  }
    60  .  .  }
    61  .  }
    62  .  Scope: *ast.Scope {
    63  .  .  Objects: map[string]*ast.Object (len = 1) {
    64  .  .  .  "main": *(obj @ 23)
    65  .  .  }
    66  .  }
    67  .  Unresolved: []*ast.Ident (len = 1) {
    68  .  .  0: *(obj @ 41)
    69  .  }
    70  .  Comments: []*ast.CommentGroup (len = 1) {
    71  .  .  0: *(obj @ 1)
    72  .  }
    73  }