Big Data Europe
目前最靠谱的样板
https://github.com/big-data-europe/docker-spark
https://github.com/big-data-europe/docker-hive
https://github.com/big-data-europe
HIVE文档
https://cwiki.apache.org/confluence/display/Hive/Home#Home-UserDocumentation
七牛日志分析
七牛日志格式
116.231.10.133 HIT 0 [07/May/2019:09:04:56 +0800] "GET http://wdl1.cache.wps.cn/per-plugin/dl/addons/pool/win-i386/wpsminisite_3.0.0.37.7z HTTP/1.1" 206 66372 "-" "-"
Qt的相关代码
int getDownloadSize(QString line) {
int idx = line.indexOf("HTTP/1.1\"");
if(idx > 0) {
QString sub = line.mid(idx);
QStringList sets = sub.split(" ");
int size = sets.at(2).toInt();
return size;
}
return 0;
}
int main(int argc, char *argv[])
{
QCoreApplication a(argc, argv);
QString dpath("F:\\日志0508\\18");
QDir d(dpath);
QStringList fs = d.entryList();
qint64 plug_total = 1;
qint64 other_total = 1;
for(int i = 0; i < fs.size(); i++) {
QString f = fs.at(i);
QFile file(dpath+"\\"+f);
if(!file.open(QIODevice::ReadOnly|QIODevice::Text)) {
continue;
}
int linecnt = 0;
while(!file.atEnd()) {
QByteArray line = file.readLine();
if(line.indexOf("wdl1.cache.wps.cn") < 0) {
continue;
}
int idx = line.indexOf("win-i386");
int size = getDownloadSize(line);
qint64 skidx = file.pos();
if(idx > 0) {
plug_total += size;
}else{
other_total += size;
}
linecnt++;
if(linecnt % 10000 == 0) {
printf("\r\nlinecnt:%d - skidx:%lld - plug_total:%lld - other_total:%lld - ratio:%5f", linecnt, skidx, plug_total, other_total, double(plug_total) / double(other_total));
}
}
printf("\r\nlinecnt:%d - plug_total:%lld - other_total:%lld - ratio:%5f", linecnt, plug_total, other_total, double(plug_total) / double(other_total));
}
printf("\r\nplug_total:%lld - other_total:%lld - ratio:%5f", plug_total, other_total, double(plug_total) / double(other_total));
return a.exec();
}
获取响应内容
log_format cntr escape=json '{"time":"$time_local",'
'"@source":"$server_addr",'
'"hostname":"$hostname",'
'"xforward":"$http_x_forwarded_for",'
'"remoteaddr":"$remote_addr",'
'"method":"$request_method",'
'"scheme":"$scheme",'
'"domain":"$server_name",'
'"referer":"$http_referer",'
'"requrl":"$request_uri",'
'"args":"$args",'
'"requestbody":"$request_body",'
'"contentlength":"$content_length",'
'"bodybytessend":$body_bytes_sent,'
'"status":$status,'
'"requesttime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamaddr":"$upstream_addr",'
'"respbody":"$resp_body",'
'"respdown":"$resp_ctrl_down",'
'"useragent":"$http_user_agent"'
'}';
location /abc {
#access_log off;
access_log /data/logs/nginx/abc/access.log cntr buffer=32k flush=5s;
error_log /data/logs/nginx/abc/error.log info;
set $resp_ctrl_down -1;
set $resp_body "";
# only elk collection machine, can add the below lua code.
body_filter_by_lua_block {
local chunk, eof = ngx.arg[1], ngx.arg[2]
ngx.var.resp_body=ngx.var.resp_body..chunk
if eof then
if string.find(ngx.var.resp_body,'"count":true') then
ngx.var.resp_ctrl_down = 1
else
ngx.var.resp_ctrl_down = 0
end
end
}
proxy_pass http://abc/flow;
}
Centos7屏蔽防火墙
systemctl stop firewalld.service
systemctl disable firewalld.service
nginx日志格式
access_log path [format [buffer=size] [gzip[=level]] [flush=time] [if=condition]]; # 设置访问日志
access_log off; # 关闭访问日志
GoLang可靠内嵌数据库
https://github.com/boltdb/bolt
fedora换阿里云镜像源
su -
cd /etc/yum.repos.d/
mv fedora.repo fedora.repo.backup
mv fedora-updates.repo fedora-updates.repo.backup
wget -O /etc/yum.repos.d/fedora.repo http://mirrors.aliyun.com/repo/fedora.repo
wget -O /etc/yum.repos.d/fedora-updates.repo http://mirrors.aliyun.com/repo/fedora-updates.repo
dnf clean all
dnf makecache
LXQT-QTermWidget的编译
LXQT的编译说明
https://github.com/lxqt/lxqt/wiki/Building-from-source
流程如下：
1.安装Qt5。
yum install qt5-qtbase-devel qt5-qtsvg-devel qt5-qttools-devel qt5-qtx11extras-devel
2.安装CMake，一定用要官方包安装，并编辑bashrc文件。
export PATH=$PATH:/cmake-3.15.3/bin
3.下载lxqt-build-tools编译工具并安装。
git clone https://github.com/lxqt/lxqt-build-tools.git
假如是解压在~/lxqt/lxqt-build-tools目录下。
则创建编译目录~/lxqt/build
构建指令如下：
cmake ../lxqt-build-tools
make && make install
它会安装至/usr/local的某个目录下。
4.下载
git clone git@github.com:kxtry/qtermwidget.git
使用CLion构建及编译即可。
此版本已经与原版不一样，有修改。
原版可能无法编译完成，原因是QString("sdfsdfds")会编译不通过，因为从Qt5.9版本开始，QString(char *p)设置成私有函数了。
终端测试验证工具
https://misc.flogisoft.com/bash/home
终端配色：https://misc.flogisoft.com/bash/tip_colors_and_formatting
colors256.sh
#!/bin/bash
for fgbg in 38 48 ; do # Foreground / Background
for color in {0..255} ; do # Colors
# Display the color
printf "\e[${fgbg};5;%sm %3s \e[0m" $color $color
# Display 6 colors per lines
if [ $((($color + 1) % 6)) == 4 ] ; then
echo # New line
fi
done
echo # New line
done
exit 0
colors_and_formatting.sh
#!/bin/bash
#Background
for clbg in {40..47} {100..107} 49 ; do
#Foreground
for clfg in {30..37} {90..97} 39 ; do
#Formatting
for attr in 0 1 2 4 5 7 ; do
#Print the result
echo -en "\e[${attr};${clbg};${clfg}m ^[${attr};${clbg};${clfg}m \e[0m"
done
echo #Newline
done
done
exit 0
——-
http://invisible-island.net/xterm/ctlseqs/ctlseqs.html
http://www.xfree86.org/current/ctlseqs.html
Linux环境下，主要都是泛VT102的终端，以下是vt102终端。
https://vt100.net/docs/vt102-ug/introduction.html
https://vt100.net/docs/vt102-ug/contents.html
微软的
https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences
VT100,VT101,
参考开源：https://konsole.kde.org/
headless vt100 emulator
https://github.com/JulienPalard/vt100-emulator
https://github.com/freanux/VTE
自动拉起服务脚本
crontab -e
*/1 * * * * sh /data/scripts/run-flow.sh start
#!/bin/sh
path_current=`pwd`
path_script=$(cd "$(dirname "$0")"; pwd)
mode=$1
logfile=/data/scripts/check.log
app_process=`ps -ef | grep "flowservice"| grep -v grep`
echo `date` >> $logfile
echo "ready to check...." >> $logfile
case "$mode" in
'start')
echo "$app_process" >> $logfile
echo "it's ready to start op...."
if test -n "$app_process"; then
echo ""
echo "$app_process"
echo ""
else
cd $path_script #进入脚本所在目录下，目的是使springboot的config目录生效。
nohup /data/code/service.flow.wps.cn/flowservice --config=/data/code/service.flow.wps.cn/config/config-prod.toml > /data/code/service.flow.wps.cn/info.txt 2>&1 &
echo "success to restart flowservice" >> $logfile
cd $path_current
fi
echo 'success to start.'
;;
'stop')
echo "it's ready to check process..."
if test -n "$app_process"; then
echo "had find app process informaton"
echo $app_process | awk '{print ($2)}' | xargs kill -9
fi
echo 'success to kill.'
;;
*)
basename=`basename "$0"`
echo "Usage: $basename {start|stop} [ server options ]"
exit 1
;;
esac
exit 1