调整项目公共内容,日志、文档基类和基础视图

1. 新增项目、机器相关模型和视图
2. 区服信息同步接口(同ops1,可接收batchQuery采集的信息)
This commit is contained in:
chenzuoqing 2021-11-30 15:55:50 +08:00
parent 0d771bf0cb
commit d07ecbc497
21 changed files with 651 additions and 31 deletions

10
app.py
View File

@ -1,15 +1,25 @@
import sys import sys
import os.path import os.path
from logging.config import dictConfig
from flask import Flask from flask import Flask
from flask.logging import default_handler
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
from database.mongodb import initialize_db from database.mongodb import initialize_db
from routes import blueprint_api from routes import blueprint_api
from settings.dev import LOGGING
# 日志配置
dictConfig(LOGGING)
# app 实例,移除默认日志,加载配置文件
app = Flask(__name__) app = Flask(__name__)
app.logger.removeHandler(default_handler)
app.config.from_pyfile("settings/dev.py") app.config.from_pyfile("settings/dev.py")
# 初始化数据库连接
initialize_db(app) initialize_db(app)
# 注册蓝图 # 注册蓝图

132
scripts/insert_test_data.py Normal file
View File

@ -0,0 +1,132 @@
from asset.models import Host
from project.models import Project, Channel, Server, Version
import logging
logger = logging.getLogger('flask.app.script')
projects = [
{
"name": "ts01/cn",
"id": 1,
"desc": "测试",
"domain": "www.baidu.com",
"www_ip": "127.0.0.1",
"ops_ip": "10.2.2.10",
"transfer_ip": "10.2.2.10",
"webhook": "",
"kw": ""
},
{
"name": "ab01/cn",
"id": 2,
"desc": "测试ab01",
"domain": "ab01.www.baidu.com",
"www_ip": "127.0.0.1",
"ops_ip": "10.2.2.11",
"transfer_ip": "10.2.2.13",
"webhook": "",
"kw": ""
},
{
"name": "aa11/tw",
"id": 3,
"desc": "1212",
"domain": "www.baidu.com",
"www_ip": "127.0.0.1",
"ops_ip": "11.22.33.44",
"transfer_ip": "11.22.33.45",
"webhook": "",
"kw": ""
},
{
"name": "qq88/cn",
"id": 4,
"desc": "qq88/cn",
"domain": "qq88.com",
"www_ip": "127.0.0.1",
"ops_ip": "10.2.2.10",
"transfer_ip": "10.2.2.11",
"webhook": "",
"kw": ""
},
{
"name": "ug41/cn",
"id": 5,
"desc": "ug41",
"domain": "ug41.huanyuantech.com",
"www_ip": "127.0.0.1",
"ops_ip": "127.0.0.1",
"transfer_ip": "127.0.0.1",
"webhook": "",
"kw": "ug41-cn"
}
]
# def sync_project():
# for p in projects:
# proj = p
# name = proj.pop("name")
# game, fork = name.split("/")
# proj["name"] = game
# proj["fork"] = fork
# proj.pop("id")
# # 字典中缺少的值会是None字段必须null=True
# obj = models.Project(**proj)
# obj.save()
def sync_server_info():
project = Project.objects(name="ug41", fork="cn").first()
info = [{"ip": "10.2.2.10", "spid": "abo", "num": 1, "server_version": "2021100801", "cfg_version": "8616", "admin_version": "2021062801", "lua_version": "122", "flag": "2", "status": "0", "port": 0}, {"ip": "10.2.2.11", "spid": "dev", "num": 1, "server_version": "2021100801", "cfg_version": "7950", "admin_version": "2021062801", "lua_version": "122", "flag": "0", "status": "1", "port": 0}, {"ip": "10.2.2.11", "spid": "dev", "num": 2, "server_version": "2021100801", "cfg_version": "2014", "admin_version": "2021062801", "lua_version": "122", "flag": "0", "status": "1", "port": 0}, {"ip": "10.2.2.11", "spid": "dev", "num": 3, "server_version": "2021100801", "cfg_version": "7003", "admin_version": "2021062801", "lua_version": "122", "flag": "0", "status": "1", "port": 0}, {"ip": "10.2.2.11", "spid": "dev", "num": 4, "server_version": "2021100801", "cfg_version": "8406", "admin_version": "2021062801", "lua_version": "122", "flag": "0", "status": "1", "port": 0}, {"ip": "10.2.2.11", "spid": "dev", "num": 5, "server_version": "2021100801", "cfg_version": "17399", "admin_version": "2021062801", "lua_version": "122", "flag": "0", "status": "1", "port": 0}, {"ip": "10.2.2.11", "spid": "dev", "num": 6, "server_version": "2021100801", "cfg_version": "8375", "admin_version": "2021062801", "lua_version": "122", "flag": "0", "status": "1", "port": 0}]
for srv in info:
ip = srv.get("ip")
num = srv.get("num")
spid = srv.get("spid")
port = srv.get("port", 0)
# 内嵌的版本字段
version = Version()
version.admin = srv.get("admin_version", "")
version.server = srv.get("server_version", "")
version.config = srv.get("cfg_version", "")
version.lua = srv.get("lua_version", "")
version.bin = srv.get("bin_version", "")
version.sql = srv.get("sql_version", "")
state = srv.get("status", "1")
if srv.get("flag", "0") == "2":
status = "closed"
else:
status = "running" if state == "1" else "error"
try:
host = Host.objects(public_ip=ip).first()
if not host:
host = Host(public_ip=ip)
host.save()
channel = Channel.objects(project=project, spid=spid).first()
if not channel:
channel = Channel(project=project, spid=spid)
channel.save()
# 更新、创建的参数
defaults = dict(host=host, version=version, port=port, status=status)
srv_obj = Server.objects(num=num, channel=channel).first()
if not srv_obj: # 创建
srv_obj = Server(num=num, channel=channel, **defaults)
srv_obj.save()
logger.info(f"创建 {spid}_{num} 区服信息成功")
continue
# 更新对象
srv_obj.update(**defaults)
srv_obj.save()
logger.info(f"更新 {spid}_{num} 区服信息成功")
except:
logger.exception("同步区服出错")
continue
if __name__ == '__main__':
# sync_project()
sync_server_info()

View File

@ -1,5 +1,47 @@
import os.path
# 项目目录
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
# mongodb 地址 # mongodb 地址
MONGODB_SETTINGS = { MONGODB_SETTINGS = {
'host': 'mongodb://admin:111111@10.2.2.10:27017/ops_api?authSource=admin', 'host': 'mongodb://admin:111111@10.2.2.10:27017/ops_api?authSource=admin',
} }
LOGGING = {
'version': 1,
'disable_existing_loggers': True,
'formatters': {
'verbose': {
'format': '%(asctime)s %(levelname)s %(module)s:%(funcName)s %(process)d %(thread)d %(message)s'
},
'simple': {
'format': '%(levelname)s %(message)s'
},
},
'filters': {
},
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'formatter': 'simple'
},
'file_app': {
'level': 'DEBUG',
'class': 'logging.handlers.RotatingFileHandler',
'filename': os.path.join(os.path.dirname(BASE_DIR), 'logs/app.log'),
'encoding': 'utf-8',
'maxBytes': 1024 * 1024 * 5,
'backupCount': 5,
'formatter': 'verbose'
},
},
'loggers': {
'root': {
'handlers': ['file_app'],
'level': 'DEBUG',
'propagate': True,
},
}
}

0
src/asset/__init__.py Normal file
View File

26
src/asset/fields.py Normal file
View File

@ -0,0 +1,26 @@
"""
asset marshal fields
"""
from flask_restful import fields
HostFields = {
"id": fields.String,
"public_ip": fields.String,
"private_ip": fields.String,
"minion_id": fields.String,
"weights": fields.Integer,
"cpu_num": fields.Integer,
"cpu_core": fields.Integer,
"memory": fields.Integer,
"tags": fields.List(fields.String),
"labels": fields.Raw,
"created": fields.DateTime,
}
HostSimpleFields = {
"id": fields.String,
"public_ip": fields.String,
"private_ip": fields.String,
}

28
src/asset/models.py Normal file
View File

@ -0,0 +1,28 @@
import mongoengine as mongo
from common.document import DocumentBase
from common.validator import is_ipaddr
class Host(DocumentBase):
# STATUS = {
# "no": ""
# }
public_ip = mongo.StringField(max_length=64, required=True, unique=True, validation=is_ipaddr)
private_ip = mongo.StringField(max_length=64, default="")
minion_id = mongo.StringField(max_length=64, default="") # 需要唯一,先留空
weights = mongo.IntField(default=40)
# status = mongo.StringField()
# spec = mongo.EmbeddedDocumentField(Spec)
cpu_num = mongo.IntField(default=1) # cpu物理个数
cpu_core = mongo.IntField(default=1) # 每个cpu的核心数
memory = mongo.IntField(default=0) # 内存大小单位GB
# 标记和标签
tags = mongo.ListField(mongo.StringField(), default=list) # tags 默认是空列表
labels = mongo.DictField(default=dict)
created = mongo.DateTimeField()

13
src/asset/routes.py Normal file
View File

@ -0,0 +1,13 @@
from flask import Blueprint
from flask_restful import Api
from asset import views
# 当前app的蓝图以app名为前缀
asset = Blueprint('asset', __name__, url_prefix="/asset")
# 增加路由
api = Api(asset)
api.add_resource(views.HostViews, '/host/', endpoint="host")
api.add_resource(views.HostDetailViews, '/host/<string:pk>/', endpoint="host-detail")

14
src/asset/views.py Normal file
View File

@ -0,0 +1,14 @@
from asset import fields
from asset.models import Host
from common.views import ListCreateViewSet, DetailViewSet
class HostViews(ListCreateViewSet):
model = Host
fields = fields.HostFields
class HostDetailViews(DetailViewSet):
model = Host
fields = fields.HostFields

44
src/common/document.py Normal file
View File

@ -0,0 +1,44 @@
"""
mongo 文档相关封装
"""
from flask_mongoengine import Document, BaseQuerySet
class DocumentBase(Document):
"""自定义的文档处理方法扩展
注意此类必须设置meta如下参考 `flask_mongoengine.Document`:
{"abstract": True, "queryset_class": BaseQuerySet}
"""
meta = {"abstract": True, "queryset_class": BaseQuerySet}
@classmethod
def get_or_create(cls, defaults: dict = None, **kwargs):
"""获取或创建一个对象,必须传好参数,确保正确且唯一,需要外部捕捉异常
:param defaults: 创建参数
:param kwargs: 对象查找参数
"""
created = False
obj = cls.objects(**kwargs).first()
if not obj:
obj = cls(**defaults)
obj.save()
created = True
return obj, created
@classmethod
def update_or_create(cls, defaults: dict = None, **kwargs):
"""更新或创建对象
:param defaults: 创建更新参数
:param kwargs: 对象查找参数
"""
created = False
obj = cls.objects(**kwargs).first()
if not obj:
obj = cls(**defaults)
obj.save()
created = True
else:
obj.update(**defaults)
return obj, created

15
src/common/validator.py Normal file
View File

@ -0,0 +1,15 @@
import ipaddress
import mongoengine as mongo
def isalnum(string: str):
"""数字、字母验证器"""
if not string.isalnum():
raise mongo.ValidationError('The value must be a combination of letters and numbers')
def is_ipaddr(ip):
try:
ipaddress.ip_address(ip)
except:
raise mongo.ValidationError('The value must be a ip address')

View File

@ -1,14 +1,14 @@
from flask_restful import abort, Resource, marshal, fields from flask_restful import abort, Resource, marshal, fields, reqparse
class ModelViewBase(Resource): class ModelViewBase(Resource):
model = None model = None
fields = {} fields = {}
request_parse = None request_parse: reqparse.RequestParser = None
def get_object(self, pk): def get_object(self, pk):
try: try:
return self.model.objects.get_or_404(id=pk) return self.model.objects(id=pk).first_or_404(message="resource not found")
except: except:
abort(404, msg=f"resource '{pk}' not found") abort(404, msg=f"resource '{pk}' not found")

View File

@ -1,2 +0,0 @@

View File

@ -1,11 +0,0 @@
from game.views import Server
from flask import Blueprint
from flask_restful import Api
# 当前app的蓝图以app名为前缀
game = Blueprint('game', __name__, url_prefix="/game")
# 增加路由
api = Api(game)
api.add_resource(Server, '/server/', endpoint="server")

View File

@ -1,12 +0,0 @@
from flask_restful import Resource
class Server(Resource):
""""""
def get(self):
return {"msg": "ok", "method": "get"}
def post(self):
return {"msg": "ok", "method": "post"}

0
src/project/__init__.py Normal file
View File

71
src/project/fields.py Normal file
View File

@ -0,0 +1,71 @@
"""
project marshal fields
"""
from flask_restful import fields
from asset.fields import HostSimpleFields
ProjectFields = {
"id": fields.String,
"name": fields.String,
"fork": fields.String,
"domain": fields.String,
"www_ip": fields.String,
"ops_ip": fields.String,
"transfer_ip": fields.String,
"webhook": fields.String,
"kw": fields.String,
"desc": fields.String,
"repository": fields.String,
"tags": fields.List(fields.String),
"labels": fields.Raw,
}
ProjectSimpleField = {
"id": fields.String,
"name": fields.String,
"fork": fields.String,
"domain": fields.String,
}
VersionFields = {
"admin": fields.String,
"server": fields.String,
"config": fields.String,
"lua": fields.String,
"sql": fields.String,
"bin": fields.String,
}
ChannelFields = {
"id": fields.String,
"project": fields.Nested(ProjectSimpleField),
"name": fields.String,
"spid": fields.String,
"version": fields.Nested(VersionFields),
"repository": fields.String,
"branch": fields.String,
"tags": fields.List(fields.String), # List 必须指明类型
"labels": fields.Raw,
}
ChannelSimpleFields = {
"id": fields.String,
"spid": fields.String,
"project": fields.Nested(ProjectSimpleField),
}
ServerFields = {
"id": fields.String,
"num": fields.Integer,
"channel": fields.Nested(ChannelSimpleFields),
"status": fields.String,
"host": fields.Nested(HostSimpleFields),
"domain": fields.String,
"port": fields.Integer,
"version": fields.Nested(VersionFields),
"weight": fields.Integer,
"slot": fields.Integer,
"tag": fields.List(fields.String),
"labels": fields.Raw,
}

88
src/project/models.py Normal file
View File

@ -0,0 +1,88 @@
import mongoengine as mongo
from common.document import DocumentBase
from common.validator import isalnum
from asset.models import Host
class Project(DocumentBase):
"""项目模型定义"""
# 四位字符字母、数字组合name、fork组合必须是唯一
name = mongo.StringField(max_length=4, min_length=4, required=True, unique_with="fork", validation=isalnum)
fork = mongo.StringField(max_length=2, min_length=2, required=True, validation=isalnum)
domain = mongo.StringField(max_length=128, required=False)
www_ip = mongo.StringField(max_length=64, required=False)
ops_ip = mongo.StringField(max_length=64, required=False)
transfer_ip = mongo.StringField(max_length=64, required=False)
webhook = mongo.StringField(max_length=256, required=False)
kw = mongo.StringField(max_length=32, required=False)
desc = mongo.StringField(max_length=256, default="")
# 项目的代码仓库地址
repository = mongo.StringField("仓库", max_length=256, default="", help_text="仓库地址", null=True)
# 标记和标签
tags = mongo.ListField(mongo.StringField(), default=list) # tags 默认是空列表
labels = mongo.DictField(default=dict)
def fullname(self):
return f"{self.name}/{self.fork}"
class Version(mongo.EmbeddedDocument):
"""版本信息"""
admin = mongo.StringField(max_length=32, required=False, default="")
server = mongo.StringField(max_length=32, required=False, default="")
config = mongo.StringField(max_length=32, required=False, default="")
lua = mongo.StringField(max_length=32, required=False, default="")
sql = mongo.StringField(max_length=32, required=False, default="")
bin = mongo.StringField(max_length=32, required=False, default="")
class Channel(DocumentBase):
"""渠道"""
# spid项目内唯一
project = mongo.ReferenceField(Project, reverse_delete_rule=mongo.NULLIFY)
name = mongo.StringField(max_length=32, default="")
spid = mongo.StringField(max_length=3, min_length=3, required=True, unique_with="project", validation=isalnum)
version = mongo.EmbeddedDocumentField(Version) # 缺失时获取对象的此字段为 None
# 项目的代码仓库地址
repository = mongo.StringField(max_length=256, default="")
branch = mongo.StringField(max_length=32)
# 标记和标签
tags = mongo.ListField(mongo.StringField(), default=list) # tags 默认是空列表
labels = mongo.DictField(default=dict)
created = mongo.DateTimeField()
class Server(DocumentBase):
"""服务"""
STATUS = {
"prepare": "预备",
"stopped": "停止",
"running": "运行中",
"closed": "关服",
"maintain": "维护",
"error": "异常"
}
num = mongo.IntField(required=True, unique_with="channel")
channel = mongo.ReferenceField(Channel)
status = mongo.StringField(max_length=12, choices=STATUS.keys(), required=True, default="running")
# 机器字段TODO 先允许为空
host = mongo.ReferenceField(Host, null=True)
domain = mongo.StringField(max_length=128, required=False)
port = mongo.IntField()
version = mongo.EmbeddedDocumentField(Version)
weight = mongo.IntField(default=1)
slot = mongo.IntField(default=0)
# 标记和标签
tags = mongo.ListField(mongo.StringField(), default=list) # tags 默认是空列表
labels = mongo.DictField(default=dict)

24
src/project/routes.py Normal file
View File

@ -0,0 +1,24 @@
from flask import Blueprint
from flask_restful import Api
from project import views
# 当前app的蓝图以app名为前缀
project = Blueprint('project', __name__, url_prefix="/project")
# 增加路由
api = Api(project)
# 项目模型的视图
api.add_resource(views.ProjectViews, '/item/', endpoint="project")
api.add_resource(views.ProjectDetailViews, '/item/<string:pk>/', endpoint="project-detail")
# 渠道模型视图
api.add_resource(views.ChannelViews, '/channel/', endpoint="channel")
api.add_resource(views.ChannelDetailViews, '/channel/<string:pk>/', endpoint="channel-detail")
# 区服模型视图
api.add_resource(views.ServerViews, '/server/', endpoint="server")
api.add_resource(views.ServerDetailView, '/server/<string:pk>/', endpoint="server-detail")
# 区服信息同步接口,更新区服信息
api.add_resource(views.ServerSyncView, '/server/sync/', endpoint="server-sync")

137
src/project/views.py Normal file
View File

@ -0,0 +1,137 @@
import datetime
from flask import current_app as app
from flask_restful import reqparse, abort
from asset.models import Host
from project import fields
from project.models import Project, Channel, Server, Version
from common.views import ListMixin, CreateMixin, ListCreateViewSet, DetailViewSet
class ProjectViews(ListMixin, CreateMixin):
model = Project
fields = fields.ProjectFields
class ProjectDetailViews(DetailViewSet):
model = Project
fields = fields.ProjectFields
class ChannelViews(ListCreateViewSet):
model = Channel
fields = fields.ChannelFields
class ChannelDetailViews(DetailViewSet):
model = Channel
fields = fields.ChannelFields
class ServerViews(ListCreateViewSet):
model = Server
fields = fields.ServerFields
class ServerDetailView(DetailViewSet):
model = Server
fields = fields.ServerFields
class ServerSyncView(CreateMixin):
"""同步区服信息,由 `batchQuery` 程序发送采集内容,此视图更新保存"""
model = Server
fields = fields.ServerFields
project: Project = None
def __init__(self):
# 参数定义json来源
self.request_parse = reqparse.RequestParser()
self.request_parse.add_argument("project", type=str, required=True, location='json')
self.request_parse.add_argument("fork", type=str, required=True, location='json')
self.request_parse.add_argument("count", type=int, required=True, location='json')
self.request_parse.add_argument("data", type=list, required=True, location='json')
def get_project(self, args):
proj = args["project"]
fork = args["fork"]
self.project = Project.objects(name=proj, fork=fork).first_or_404(message="project not found")
@classmethod
def parse_version(cls, srv: dict):
"""解析版本"""
version = Version()
version.admin = srv.get("admin_version", "")
version.server = srv.get("server_version", "")
version.config = srv.get("cfg_version", "")
version.lua = srv.get("lua_version", "")
version.bin = srv.get("bin_version", "")
version.sql = srv.get("sql_version", "")
return version
@classmethod
def parse_status(cls, srv: dict):
"""解析 flag 和 status 值,据此判断返回区服的状态"""
state = srv.get("status", "1")
if srv.get("flag", "0") == "2":
status = "closed"
else:
status = "running" if state == "1" else "error"
return status
def post(self):
"""同步区服信息,接收 json 数据"""
# 校验参数
args = self.request_parse.parse_args()
self.get_project(args)
count = args["count"]
data = args["data"]
if count != len(data):
abort(400, msg="quantity mismatch", code=1001)
hosts = {}
channels = {}
errors = []
for srv in data:
ip = srv.get("ip")
num = srv.get("num")
spid = srv.get("spid")
port = srv.get("port", 0)
# 内嵌的版本字段
version = self.parse_version(srv)
status = self.parse_status(srv)
host = hosts.get(ip)
channel = channels.get(spid)
try:
if not host:
hostObj, _ = Host.get_or_create(
public_ip=ip, defaults=dict(public_ip=ip, created=datetime.datetime.now()))
host = hostObj.id
hosts[ip] = hostObj.id
if not channel:
channelObj, _ = Channel.get_or_create(
project=self.project, spid=spid, defaults=dict(project=self.project, spid=spid))
channel = channelObj.id
channels[spid] = channelObj.id
# 更新、创建的参数
defaults = dict(num=num, channel=channel, host=host, version=version, port=port, status=status)
obj, created = Server.update_or_create(defaults=defaults, num=num, channel=channel)
if created: # 创建
app.logger.debug(f"创建 {spid}_{num} 区服信息成功")
continue
# 更新对象日志
app.logger.debug(f"更新 {spid}_{num} 区服信息成功")
except:
app.logger.exception("同步区服出错")
errors.append(f"{spid}_s{num}")
continue
if errors:
return {"msg": "有区服同步出错!", "code": 1020, "errors": errors}
return {"msg": "ok", "code": 1000}

View File

@ -1,9 +1,10 @@
from flask import Blueprint from flask import Blueprint
from game.routes import game from project.routes import project
from asset.routes import asset
blueprint_api = Blueprint('api-main', __name__, url_prefix='/api') blueprint_api = Blueprint('api-main', __name__, url_prefix='/api')
# 注册子蓝图,嵌套 # 注册子蓝图,嵌套
blueprint_api.register_blueprint(game) blueprint_api.register_blueprint(asset)
blueprint_api.register_blueprint(project)