chenzuoqing
/
ops-api
Template
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

中间件、实例模型和视图调整

This commit is contained in:
chenzuoqing 2021-12-28 16:21:53 +08:00
parent 065aa9d34c
commit 3ad65d90ca
12 changed files with 198 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/common/fields.py
View File

@ -1,4 +1,4 @@
from flask_restful import fields
from flask_restful import fields, marshal
class DateTime(fields.DateTime):
@ -18,3 +18,20 @@ class DateTime(fields.DateTime):
return value.strftime(self.dt_format)
except AttributeError as ae:
raise fields.MarshallingException(ae)
class NestedOrEmpty(fields.Nested):
"""若嵌套信息为空，不返回嵌套结构全部字段，可仅返回空字典"""
def __init__(self, nested, allow_empty=False, **kwargs):
self.allow_empty = allow_empty
super().__init__(nested, **kwargs)
def output(self, key, obj):
value = fields.get_value(key if self.attribute is None else self.attribute, obj)
if value is None:
if self.allow_null:
return None
elif self.allow_empty:
return {}
return marshal(value, self.nested)

18
src/common/views.py
View File

@ -44,6 +44,7 @@ class ModelViewBase(Resource):
model = None
# marshal 字段
fields = {}
fields_detail = {}
# 参数解析校验
request_parse: reqparse.RequestParser = None
# 是否分页
@ -153,9 +154,13 @@ class ModelViewBase(Resource):
for field, model, required in self.relation_fields:
val = args.get(field)
# 判断 required: 参数有提交值才校验，未在参数中或零值不校验
if not required and not val:
if (not required or required == "many") and not val:
continue
if required == "many" and isinstance(val, list):
for v in val:
# 遍历列表检查是否存在
self.validate_relation_pk(v, model, field)
return
self.validate_relation_pk(val, model, field)
@ -203,14 +208,19 @@ class ListMixin(ModelViewBase):
# 过滤后的数据，
self.get_queryset(*args, **kwargs)
self.queryset = self.filter_queryset()
count = self.queryset.count()
# 根据请求头返回不同解析的fields
if request.headers.get("Accept") == "salt":
field_parse = self.fields_detail if self.fields_detail else self.fields
return formatter.key_indexed_map(self.queryset, field_parse)
limit = request.args.get("limit")
if self.paging and limit != "0":
return self.paginate_queryset(self. queryset)
# 转为对象列表
if self.queryset.count() > 0:
if count > 0:
return formatter.to_list(self.queryset, self.fields, only_exist=True)
# return marshal(self.queryset, self.fields, envelope="id")
# return formatter.key_indexed_map(self.queryset, self.fields)

2
src/controller/asset/host.py
View File

@ -4,7 +4,7 @@ import logging
from flask import request
from flask_restful import reqparse
from models.asset import fields as assetField
from models.asset import hostFields as assetField
from models.asset import host as assetModel
from common.views import ListCreateViewSet, DetailViewSet
from common.permission import session_or_token_required

18
src/controller/asset/instance.py
View File

@ -1,6 +1,7 @@
from flask import request
from flask_restful import reqparse
from models.asset.credential import Credential
from models.asset.middleware import Middleware
from models.asset.instance import Instance
from models.asset.instanceFields import InstanceFields, InstanceDetailFields
@ -15,7 +16,9 @@ class InstanceParse:
"""
request_parse = None
# 给子类用的唯一字段，用于校验
uniq_fields = (("name", "class_name"),)
uniq_fields = (("name", "server"), )
# 需要检查的关联字段对象
relation_fields = (("server", Middleware, ""), ("credential", Credential, "many"), ("dependents", Middleware, "many"))
def __init__(self):
# 创建时必要的参数用此变量判断，更新默认都可以不传
@ -23,7 +26,9 @@ class InstanceParse:
self.request_parse = reqparse.RequestParser()
self.request_parse.add_argument("name", required=required, type=str, location='json')
self.request_parse.add_argument("class_name", required=False, type=str, location='json')
self.request_parse.add_argument("middleware_id", required=False, type=str, location='json')
self.request_parse.add_argument("server", required=False, type=str, location='json')
self.request_parse.add_argument("dependents", required=False, type=list, location='json')
self.request_parse.add_argument("credentials", required=False, type=list, location='json')
self.request_parse.add_argument("extra", required=False, type=dict, location='json')
@ -42,14 +47,6 @@ class InstanceClassNameValidate:
if class_name_params and class_name_params != class_name:
return abort_response(400, 1400, msg="URL中的类别与内容中的类别字段有差异!")
args["class_name"] = class_name
# 校验 middleware_id 若设置值，检查 middleware 对象是否存在，且类型与 instance 一致
middleware_id = args.get("middleware_id")
if middleware_id:
try:
assert Middleware.objects(id=middleware_id, class_name=class_name).first()
except:
return abort_response(400, 1400, msg="参数中的同类型 middleware 对象不存在")
return args
@ -68,6 +65,7 @@ class InstanceClassViews(InstanceParse, InstanceClassNameValidate, ListCreateVie
"""某类别的实例列表"""
model = Instance
fields = InstanceFields
fields_detail = InstanceDetailFields
def get_queryset(self, *args, **kwargs):
self.queryset = self.model.objects(**kwargs)

38
src/controller/asset/middleware.py
View File

@ -1,8 +1,10 @@
from flask import request
from flask_restful import reqparse
from models.asset.middleware import Middleware
from models.asset.middlewareFields import MiddlewareFields
from models.asset.host import Host
from models.asset.credential import Credential
from models.asset.middleware import Middleware, Network
from models.asset.middlewareFields import MiddlewareFields, MiddlewareDetailFields
from common.utils import abort_response
from common.views import (ListCreateViewSet, ListMixin, DetailViewSet,
RetrieveClassMixin, DestroyClassMixin, UpdateClassMixin)
@ -13,19 +15,23 @@ class MiddlewareParse:
中间件模型公共字段的校验类
"""
request_parse = None
# 给子类用的唯一字段，用于校验
uniq_fields = ("name", "host")
# uniq_fields = ("name",)
relation_fields = (("server", Host, ""), ("credential", Credential, "many"))
def init_parse(self):
def __init__(self):
# 创建时必要的参数用此变量判断，更新默认都可以不传
required = request.method == "POST"
self.request_parse = reqparse.RequestParser()
self.request_parse.add_argument("name", required=required, type=str, location='json')
self.request_parse.add_argument("class_name", required=False, type=str, location='json')
self.request_parse.add_argument("host", required=False, type=str, location='json')
self.request_parse.add_argument("port", required=False, type=int, location='json')
self.request_parse.add_argument("manage", required=False, type=str, location='json')
self.request_parse.add_argument("network", required=False, type=list, location='json')
self.request_parse.add_argument("capacity", required=False, type=int, location='json')
self.request_parse.add_argument("version", required=False, type=str, location='json')
self.request_parse.add_argument("status", type=str, location='json', choices=Middleware.STATUS.keys())
self.request_parse.add_argument("server", required=False, type=str, location='json')
self.request_parse.add_argument("credentials", required=False, type=list, location='json')
self.request_parse.add_argument("extra", required=False, type=dict, location='json')
self.request_parse.add_argument("data", required=False, type=dict, location='json')
@ -41,28 +47,31 @@ class MiddlewareClassNameValidate:
if class_name_params and class_name_params != class_name:
return abort_response(400, 1400, msg="URL中的类别与内容中的类别字段有差异!")
args["class_name"] = class_name
network = args.get("network", [])
if network:
network_list = [Network(**n) for n in network if isinstance(n, dict)]
args["network"] = network_list
return args
class MiddlewareViews(MiddlewareParse, ListCreateViewSet):
model = Middleware
fields = MiddlewareFields
uniq_fields = ("name",)
filter_fields = (("name", "icontains"), ("host", "icontains"), ("manage", "icontains"), ("class_name", ""))
filter_fields = (("name", "icontains"), ("class_name", ""))
class MiddlewareDetail(MiddlewareParse, DetailViewSet):
"""分类别的视图"""
model = Middleware
fields = MiddlewareFields
uniq_fields = ("name",)
fields = MiddlewareDetailFields
class MiddlewareClassViews(MiddlewareParse, MiddlewareClassNameValidate, ListCreateViewSet):
"""按分类的中间件视图，带 class_name 参数"""
model = Middleware
fields = MiddlewareFields
uniq_fields = ("name",)
fields_detail = MiddlewareDetailFields
filter_fields = (("name", "icontains"), ("host", "icontains"), ("manage", "icontains"))
def get_queryset(self, *args, **kwargs):
@ -74,5 +83,4 @@ class MiddlewareClassDetail(MiddlewareParse, MiddlewareClassNameValidate,
RetrieveClassMixin, DestroyClassMixin, UpdateClassMixin):
"""分类别的详情视图，带 class_name 和 pk 参数"""
model = Middleware
fields = MiddlewareFields
uniq_fields = ("name",)
fields = MiddlewareDetailFields

2
src/controller/project/operation.py
View File

@ -4,7 +4,7 @@ import datetime
from flask_restful import Resource, reqparse, marshal, fields as F
from models.asset.host import Host
from models.asset.fields import HostFields
from models.asset.hostFields import HostFields
from models.project import fields
from models.project.models import Project, Channel, Server, Version
from common.views import CreateMixin, RetrieveMixin

0
src/models/asset/fields.py → src/models/asset/hostFields.py
View File

46
src/models/asset/instance.py
View File

@ -2,34 +2,52 @@ import mongoengine as mongo
from common.document import DocumentBase
from models.asset.middleware import Middleware
from models.asset.credential import Credential
class Instance(DocumentBase):
""""""
meta = {'allow_inheritance': True, 'collection': 'instance'}
"""实例集合"""
STATUS = {
# TODO 完善状态
"none": "未规划", # 默认状态
"prepare": "待部署",
"normal": "正常",
}
meta = {'allow_inheritance': True, 'collection': 'instance', 'strict': False}
# 实例的分类名称
class_name = mongo.StringField(required=False, default="")
# 实例的名称，可能是库名、仓库名等
name = mongo.StringField(required=True)
status = mongo.StringField(choices=STATUS.keys(), default="none")
# 关联的 middleware
middleware_id = mongo.ObjectIdField(required=False)
# 关联的 middleware（是当前实例的父级）
server = mongo.ObjectIdField(required=False)
# 依赖的 middleware 列表
dependents = mongo.ListField(mongo.ObjectIdField(), default=list)
# 对应可以有多个凭据
credentials = mongo.ListField(mongo.ObjectIdField, default=list)
credentials = mongo.ListField(mongo.ObjectIdField(), default=list)
# 按类型不同的信息存放到 extra 字典
extra = mongo.DictField(default=dict)
data = mongo.DictField(default=dict)
tags = mongo.ListField(mongo.StringField(), default=list) # tags 默认是空列表
labels = mongo.DictField(default=dict)
@property
def middleware(self):
if self.middleware_id:
try:
return Middleware.objects(id=self.middleware_id).first()
except:
pass
return None
def server_object(self):
if self.server:
return Middleware.objects(id=self.server).first()
return
@property
def dependents_object(self):
if self.dependents:
return Middleware.objects(id__in=self.dependents)
return
@property
def credentials_object(self):
if self.credentials:
return Credential.objects(id__in=self.credentials)
return

13
src/models/asset/instanceFields.py
View File

@ -7,7 +7,11 @@ InstanceFields = {
"id": fields.String,
"name": fields.String,
"class_name": fields.String,
"middleware_id": fields.String,
"status": fields.String,
"server": fields.String,
"dependents": fields.List(fields.String),
"credentials": fields.List(fields.String),
"extra": fields.Raw,
"data": fields.Raw,
@ -20,8 +24,11 @@ InstanceDetailFields = {
"id": fields.String,
"name": fields.String,
"class_name": fields.String,
"middleware": fields.Nested(MiddlewareFields),
"credentials": fields.List(fields.String),
"status": fields.String,
"server": fields.Nested(MiddlewareFields, allow_null=True, attribute="server_object"), # 若没有关联信息返回 null
"dependents": fields.List(fields.Nested(MiddlewareFields), attribute="dependents_object"),
"credentials": fields.List(fields.Nested(MiddlewareFields), attribute="credentials_object"),
"extra": fields.Raw,
"data": fields.Raw,

65
src/models/asset/middleware.py
View File

@ -1,33 +1,70 @@
import mongoengine as mongo
from common.document import DocumentBase
from models.asset.host import Host
from models.asset.credential import Credential
class Network(mongo.EmbeddedDocument):
"""网络信息"""
# 地址 端口 是否对外
address = mongo.StringField(default="")
port = mongo.IntField(default=-1)
class Middleware(DocumentBase):
"""中间件"""
meta = {'allow_inheritance': True, 'collection': 'mdw', 'strict': False}
STATUS = {
# TODO 完善状态
"none": "未规划", # 默认状态
"prepare": "待部署",
"normal": "正常",
}
meta = {'allow_inheritance': True, 'collection': 'middleware', 'strict': False}
# 分类名称
class_name = mongo.StringField(required=False, default="")
name = mongo.StringField(required=True)
host = mongo.StringField(required=False, default="")
port = mongo.IntField(default=0)
# 管理者，预留
manage = mongo.StringField(max_length=128, required=False, default="")
# 网络，监听地址和端口，可能多个
network = mongo.EmbeddedDocumentListField(Network)
# 容量
capacity = mongo.IntField(default=-1)
status = mongo.StringField(choices=STATUS.keys(), default="none")
# 中间件程序的版本
version = mongo.StringField(default="")
# 服务器，对应 Host
server = mongo.ObjectIdField(required=False)
# 对应可以有多个凭据
credentials = mongo.ListField(mongo.ObjectIdField(), default=list)
extra = mongo.DictField(default=dict)
data = mongo.DictField(default=dict)
# 标记和标签
tags = mongo.ListField(mongo.StringField(), default=list) # tags 默认是空列表
labels = mongo.DictField(default=dict)
@property
def minion(self):
"""返回机器的 minion_id"""
if self.server:
host: Host = self.server_object
if host:
return host.minion_id
return
# class DBServer(Middleware):
# """数据库服务器"""
# meta = {'allow_inheritance': True, 'strict': False}
# domain = mongo.StringField(required=False) # 域名连接的地址
# credentials = mongo.ListField(mongo.ObjectIdField, default=list)
@property
def server_object(self):
"""返回 host 对象"""
if self.server:
try:
return Host.objects(id=self.server).first()
except:
pass
return
# 配置
# storage = mongo.IntField(required=False, default=0)
# memory = mongo.IntField(required=False, default=0)
# core = mongo.IntField(required=False, default=0)
@property
def credentials_object(self):
if self.credentials:
return Credential.objects(id__in=self.credentials)
return

65
src/models/asset/middlewareFields.py
View File

@ -1,38 +1,51 @@
from flask_restful import fields
from models.asset.hostFields import HostFields
from models.asset.credentialFields import CredentialFields
NetworkFields = {
"address": fields.String,
"port": fields.Integer,
}
# 中间件列表解析字段
MiddlewareFields = {
"id": fields.String,
"name": fields.String,
"host": fields.String,
"port": fields.Integer,
"manage": fields.String,
"class_name": fields.String,
"extra": fields.Raw,
"network": fields.List(fields.Nested(NetworkFields)),
"capacity": fields.Integer,
"status": fields.String,
"version": fields.String,
"server": fields.String,
"credentials": fields.List(fields.String),
"extra": fields.Raw,
"data": fields.Raw,
"tags": fields.List(fields.String),
"labels": fields.Raw,
}
# DatabaseFields = {
# "id": fields.String,
# "name": fields.String,
# "username": fields.String,
# "password": fields.String,
# "data": fields.Raw,
# "tags": fields.List(fields.String),
# "labels": fields.Raw,
# }
#
# DBServerFields = {
# "domain": fields.String,
# "port": fields.Integer,
# "username": fields.String,
# "password": fields.String,
# "storage": fields.Integer,
# "memory": fields.Integer,
# "core": fields.Integer,
# "databases": fields.List(fields.Nested(DatabaseFields))
# }
#
# DBServerFields.update(MiddlewareFields)
# 中间件详情解析的字段
MiddlewareDetailFields = {
"id": fields.String,
"name": fields.String,
"class_name": fields.String,
"network": fields.List(fields.Nested(NetworkFields)),
"capacity": fields.Integer,
"status": fields.String,
"version": fields.String,
"minion": fields.String(attribute="minion"),
"server": fields.Nested(HostFields, attribute="server_object", allow_null=True),
"credentials": fields.List(fields.Nested(CredentialFields), attribute="credentials_object"),
"extra": fields.Raw,
"data": fields.Raw,
"tags": fields.List(fields.String),
"labels": fields.Raw,
}

2
src/models/project/fields.py
View File

@ -3,7 +3,7 @@ project marshal fields
"""
from flask_restful import fields
from models.asset.fields import HostSimpleFields
from models.asset.hostFields import HostSimpleFields
ProjectFields = {
"id": fields.String,