Skip to content

Commit

Permalink
Merge pull request #10 from TTB-Network/dev/stats
Browse files Browse the repository at this point in the history
feat: Add Statistics for files, request per seconds...
  • Loading branch information
SilianZ authored Mar 10, 2024
2 parents 703b4c8 + cb71832 commit f1cb130
Show file tree
Hide file tree
Showing 11 changed files with 433 additions and 73 deletions.
9 changes: 5 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,6 @@

🎉 __*新增功能!*__ 基于 Echart 的 OpenBMCLAPI 仪表盘(Dashboard)。

🎉 __*新增功能!*__ 基于 loguru 的**日志器**


</div>

# 简介
Expand Down Expand Up @@ -109,10 +106,14 @@ web_port: 8800
web_publicport: 8800
```

# 贡献

如果你有能力,你可以向我们的团队[发送邮件](mailto://administrator@ttb-network.top)或团队所有者[发送邮件](mailto://silian_zheng@outlook.com)并申请加入开发者行列。

# 鸣谢

[LiterMC/go-openbmclapi](https://github.com/LiterMC/go-openbmclapi)

[bangbang93/openbmclapi](https://github.com/bangbang93/openbmclapi)

[SALTWOOD/CSharp-OpenBMCLAPI](https://github.com/SALTWOOD/CSharp-OpenBMCLAPI)
[SALTWOOD/CSharp-OpenBMCLAPI](https://github.com/SALTWOOD/CSharp-OpenBMCLAPI)
99 changes: 96 additions & 3 deletions container/bmclapi_dashboard/static/js/index.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
const UNIT_BYTES = [
"", "K", "M", "G", "T", "E"
"K", "M", "G", "T", "E"
];
const calc_bits = (v) => {
v *= 8
Expand Down Expand Up @@ -112,10 +112,9 @@ const calc_more_bytes = (...values) => {
axios.get("master?url=/openbmclapi/metric/dashboard").then(response => {
if (response.status != 200) return
data = response.data
console.log(data)
document.getElementById("t-clusters-nodes").innerText = data.currentNodes
document.getElementById("t-clusters-bandwidth").innerText = data.currentBandwidth.toFixed(2) + " M"
document.getElementById("t-clusters-bytes").innerText = calc_bytes(data.bytes * 1024.0)
document.getElementById("t-clusters-bytes").innerText = calc_bytes(data.bytes)
document.getElementById("t-clusters-req").innerText = (data.hits / 10000).toFixed(2)
nodes = []
bytes = []
Expand Down Expand Up @@ -203,6 +202,100 @@ const calc_more_bytes = (...values) => {
).childWidth("33.33%", "33.33%", "33.33%").valueOf()
]
}
},
"dashboard": {
"connect": () => {
if (!("dashboard" in core_modules_locals)) {
core_modules_locals["dashboard"] = {
"refresh": () => {
axios.get("/dashboard").then(resp => {
if (resp.status != 200) return
data = resp.data
req = Array.from({ length: 24 }, (_, __) => null)
hits = Array.from({ length: 24 }, (_, __) => null)
bandwidth = Array.from({ length: 24 }, (_, __) => null)
bytes = Array.from({ length: 24 }, (_, __) => null)
days = data.days[0]
for (day of data.days) {
if (days._day < day._day)
days = day
}
for (hourly of data.hourly) {
const hour = hourly._hour
req[hour] = (hourly.qps / 10000).toFixed(2)
hits[hour] = (hourly.hits / 10000).toFixed(2)
bandwidth[hour] = (hourly.bandwidth * 8 / 1024.0 / 1024.0).toFixed(2)
bytes[hour] = (hourly.bytes / 1024.0 / 1024.0 / 1024.0).toFixed(2)
}
core_modules_locals["dashboard"]["req"] .setOption({title: {text: "每小时请求分布(万)"}, tooltip:{formatter: e => e[0].data == null ? '' : '<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">请求: </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'万</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: req}]})
core_modules_locals["dashboard"]["bytes"] .setOption({title: {text: "每小时流量分布(GiB)"}, tooltip:{formatter: e => e[0].data == null ? '' : '<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">流量: </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'GiB</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: bytes}]})
core_modules_locals["dashboard"]["hit"] .setOption({title: {text: "每小时请求文件数(万)"}, tooltip:{formatter: e => e[0].data == null ? '' : '<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">请求文件: </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'万</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: hits}]})
core_modules_locals["dashboard"]["bandwidth"] .setOption({title: {text: "每小时峰值出网带宽(Mbps)"}, tooltip:{formatter: e => e[0].data == null ? '' : '<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">带宽: </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'Mbps</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: bandwidth}]})
document.getElementById("t-d-req").innerText = (days.qps / 10000).toFixed(2)
document.getElementById("t-d-bytes").innerText = calc_bytes(days.bytes)
document.getElementById("t-d-hit").innerText = (days.hit / 10000).toFixed(2)
document.getElementById("t-d-bandwidth").innerText = (days.bandwidth * 8 / 1024.0 / 1024.0).toFixed(2) + " M"
})
},
"bandwidth": echarts.init(document.getElementById("e-d-bandwidth")),
"bytes": echarts.init(document.getElementById("e-d-bytes")),
"req": echarts.init(document.getElementById("e-d-req")),
"hit": echarts.init(document.getElementById("e-d-hit")),
"load": echarts.init(document.getElementById("e-d-cpu")),
"options": {tooltip:{trigger:"axis",axisPointer:{type:"cross",label:{backgroundColor:"#0FC6C2"}}},grid:{left:"3%",right:"4%",bottom:"3%",containLabel:!0},xAxis:{type:"category",boundaryGap:!1,data:time_hours},yAxis:{type:"value",axisLabel:{formatter:"{value}"}},series:[{name:"",type:"line",stack:"",areaStyle:{},color:"#0FC6C2",symbol:"circle",symbolSize:4,data:[],smooth:!0,animationEasing:"cubicOut",animationDelay:function(t){return 10*t}}]},
}
core_modules_locals["dashboard"]["bandwidth"].setOption(core_modules_locals["dashboard"]["options"])
core_modules_locals["dashboard"]["bytes"] .setOption(core_modules_locals["dashboard"]["options"])
core_modules_locals["dashboard"]["req"] .setOption(core_modules_locals["dashboard"]["options"])
core_modules_locals["dashboard"]["hit"] .setOption(core_modules_locals["dashboard"]["options"])
}
core_modules_locals["dashboard"]["timer"] = setInterval(core_modules_locals["dashboard"].refresh, 30000)
core_modules_locals["dashboard"].refresh()
},
"page": () => [
ExtendFlex().append(
ExtendElement("div").append(
ExtendElement("div").css("panel").append(
ExtendElement("h4").text("当日出网峰值带宽").valueOf(),
ExtendElement("h2").append(
ExtendElement("span").text("0 ").id("t-d-bandwidth").valueOf(),
ExtendElement("span").text("bps").valueOf()
).valueOf(),
ExtendElement("div").id("e-d-bandwidth").style("height: 216px; width: 100%").valueOf()
).valueOf(),
ExtendElement("div").css("panel").append(
ExtendElement("h4").text("当日请求文件数").valueOf(),
ExtendElement("h2").append(
ExtendElement("span").text("0 ").id("t-d-hit").valueOf(),
ExtendElement("span").text("万").valueOf()
).valueOf(),
ExtendElement("div").id("e-d-hit").style("height: 216px; width: 100%").valueOf()
).valueOf(),
),
ExtendElement("div").append(
ExtendElement("div").css("panel").append(
ExtendElement("h4").text("当日总流量").valueOf(),
ExtendElement("h2").append(
ExtendElement("span").text("0 ").id("t-d-bytes").valueOf(),
ExtendElement("span").text("iB").valueOf()
).valueOf(),
ExtendElement("div").id("e-d-bytes").style("height: 216px; width: 100%").valueOf()
).valueOf(),
ExtendElement("div").css("panel").append(
ExtendElement("h4").text("当日请求数").valueOf(),
ExtendElement("h2").append(
ExtendElement("span").text("0 ").id("t-d-req").valueOf(),
ExtendElement("span").text("万").valueOf()
).valueOf(),
ExtendElement("div").id("e-d-req").style("height: 216px; width: 100%").valueOf()
).valueOf(),
),
ExtendElement("div").css("panel").append(
ExtendElement("h4").text("五分钟负载").valueOf(),
ExtendElement("div").id("e-d-cpu").style("height: 98%; width: 100%").valueOf()
),
).childWidth("33.33%", "33.33%", "33.33%").valueOf()
]
}
}
const handler = ((root, key, type) => {
Expand Down
30 changes: 20 additions & 10 deletions container/cluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,15 @@
import utils
import stats
import web
from logger import logger
import logger
from tqdm import tqdm

PY_VERSION = "1.0.0"
VERSION = "1.9.7"
UA = f"openbmclapi-cluster/{VERSION} Python/{PY_VERSION}"
URL = 'https://openbmclapi.bangbang93.com/'
COUNTER = stats.Counters()

COUNTER = stats.counter
LAST_COUNTER = stats.last_counter
@dataclass
class BMCLAPIFile:
path: str
Expand Down Expand Up @@ -127,7 +127,7 @@ async def check_file(self):
total = len(filelist)
byte = 0
miss = []
pbar = tqdm(total=total, unit=' file(s)', unit_scale=True)
pbar = tqdm(file=logger.PRINTSTDOUT, total=total, unit=' file(s)', unit_scale=True)
pbar.set_description("Checking files")
for i, file in enumerate(filelist):
filepath = str(self.dir) + f"/{file.hash[:2]}/{file.hash}"
Expand Down Expand Up @@ -214,12 +214,12 @@ async def message(self, type, data):
logger.error("Error:" + data[0]['message'])
Timer.delay(self.enable)
elif type == "keep-alive":
COUNTER.hit -= self.cur_counter.hit
COUNTER.bytes -= self.cur_counter.bytes
LAST_COUNTER.hit += self.cur_counter.hit
LAST_COUNTER.bytes += self.cur_counter.bytes
self.keepalive = Timer.delay(self.keepaliveTimer, (), 5)
async def keepaliveTimer(self):
self.cur_counter.hit = COUNTER.hit
self.cur_counter.bytes = COUNTER.bytes
self.cur_counter.hit = COUNTER.hit - LAST_COUNTER.hit
self.cur_counter.bytes = COUNTER.bytes - LAST_COUNTER.bytes
await self.emit("keep-alive", {
"time": time.time(),
"hits": self.cur_counter.hit,
Expand Down Expand Up @@ -272,7 +272,7 @@ async def __call__(self) -> io.BytesIO:
if self.size == stat.st_size and self.last_file == stat.st_mtime:
self.last = time.time() + 1440
return self.buf
self.buf.seek(0, os.SEEK_SET)
self.buf = io.BytesIO()
async with aiofiles.open(self.file, "rb") as r:
while (data := await r.read(min(config.IO_BUFFER, stat.st_size - self.buf.tell()))) and self.buf.tell() < stat.st_size:
self.buf.write(data)
Expand All @@ -288,6 +288,8 @@ async def init():
global storage
Timer.delay(storage.check_file)
app = web.app
def record_bandwidth(sent: int, recv: int):
COUNTER.bandwidth += sent
@app.get("/measure/{size}")
async def _(request: web.Request, size: int, s: str, e: str):
#if not config.SKIP_SIGN:
Expand All @@ -302,12 +304,14 @@ async def _(request: web.Request, hash: str, s: str, e: str):
#if not config.SKIP_SIGN:
# check_sign(request.protocol + "://" + request.host + request.path, config.CLUSTER_SECRET, s, e)
file = Path(str(storage.dir) + "/" + hash[:2] + "/" + hash)
COUNTER.qps += 1
if not file.exists():
return web.Response(status_code=404)
if hash not in cache:
cache[hash] = FileCache(file)
data = await cache[hash]()
COUNTER.bytes += len(data.getbuffer())
COUNTER.bytes += cache[hash].size
request.client.set_log_network(record_bandwidth)
COUNTER.hit += 1
return data.getbuffer()
router: web.Router = web.Router("/bmcl")
Expand All @@ -324,6 +328,12 @@ async def _(request: web.Request, url: str):
async with session.get(url) as resp:
content.write(await resp.read())
return content # type: ignore
@router.get("/dashboard")
async def _():
return {
"hourly": stats.hourly(),
"days": stats.days()
}
app.mount(router)

async def clearCache():
Expand Down
2 changes: 1 addition & 1 deletion container/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ def __init__(self, path: str) -> None:

def load(self):
with open(self.file, "r", encoding="utf-8") as f:
self.cfg = yaml.load(f.read(), Loader=yaml.FullLoader)
self.cfg = yaml.load(f.read(), Loader=yaml.FullLoader) or {}

def get(self, key, default_):
value = self.cfg.get(key, default_)
Expand Down
49 changes: 46 additions & 3 deletions container/logger.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,47 @@
from loguru import logger
from pathlib import Path
from enum import Enum
import inspect
import io
import sys
import time
STDOUT = sys.stdout
class Stdout(io.StringIO):
def write(self, __s: str) -> int:
return STDOUT.write(__s)
def flush(self) -> None:
return STDOUT.flush()
def seek(self, __cookie: int, __whence: int = 0) -> int:
return STDOUT.seek(__cookie, __whence)
sys.stdout = Stdout()

logger.add(Path("./logs/{time}.log"), rotation="3 hours")
class PrintStdout(io.StringIO):
def write(self, __s: str) -> int:
info(__s.lstrip("\r"), flush=True)
return len(__s)
PRINTSTDOUT = PrintStdout()

class Level(Enum):
DEBUG = 0
INFO = 1
WARNING = 2
ERROR = 3
LevelColors: dict[Level, str] = {
Level.DEBUG: "reset",
Level.INFO: "green",
Level.WARNING: "yellow",
Level.ERROR: "red"
}

def logger(*values, level: Level, flush: bool = False, stack: list[inspect.FrameInfo]):
print(*(f"<<<flush:{flush},time:{time.time()},color:{LevelColors.get(level, 'reset')}>>>[{level.name.upper()}]", *values))

def info(*values, flush: bool = False):
return logger(*values, flush=flush, level=Level.INFO, stack=inspect.stack())

def error(*values, flush: bool = False):
return logger(*values, flush=flush, level=Level.ERROR, stack=inspect.stack())

def warning(*values, flush: bool = False):
return logger(*values, flush=flush, level=Level.WARNING, stack=inspect.stack())

def debug(*values, flush: bool = False):
return logger(*values, flush=flush, level=Level.DEBUG, stack=inspect.stack())
6 changes: 6 additions & 0 deletions container/main.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
from datetime import datetime
import os
import time
cur = time.time()
os.environ["UTC"] = str(int((datetime.fromtimestamp(cur) - datetime.utcfromtimestamp(cur)).total_seconds() / 3600))

if __name__ == "__main__":
import web
web.init()
Loading

0 comments on commit f1cb130

Please sign in to comment.