init

web/commons/constants/__init__.py

@@ -0,0 +1 @@
+from . import *

web/commons/constants/material_task.py

@@ -0,0 +1,21 @@
+class MaterialTaskType:
+    OTHER_ZHEJIANG = 601  # 浙江
+    OTHER_GUANGZHOU = 602  # 广州
+    OTHER_YUNNAN = 603  # 云南
+    FUJIAN_DEPARTMENT = 701  # 住建厅
+    OIL = 801  # 汽柴油
+
+    values = {
+        OTHER_ZHEJIANG: '浙江',
+        OTHER_GUANGZHOU: '广州',
+        OTHER_YUNNAN: '云南',
+        FUJIAN_DEPARTMENT: '住建厅',
+        OIL: '汽柴油',
+    }
+
+
+class MaterialTaskStatus:
+    WAITING = 0
+    DOING = 1
+    DONE = 2
+    FAILED = -1

web/commons/constants/mysteel.py

@@ -0,0 +1,16 @@
+
+class PageType:
+    REBAR_LIST = '福州建筑钢材价格行情'
+    REBAR_DETAIL = '福州建筑钢材价格行情详情'
+    PLATE_LIST = '福州中板价格行情'
+    PLATE_DETAIL = '福州中板价格行情详情'
+    SECTION_LIST = '福州型钢价格行情'
+    SECTION_DETAIL = '福州型钢价格行情详情'
+    SHAGANG_LIST = '沙钢调价信息'
+    SHAGANG_DETAIL = '沙钢调价信息详情'
+    SANGANG_LIST = '三钢调价信息'
+    SANGANG_DETAIL = '三钢调价信息详情'
+
+
+
+

web/commons/helpers/auth.py

@@ -0,0 +1,9 @@
+from functools import wraps
+
+
+def basic_auth_required(f):
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        return f(*args, **kwargs)
+
+    return decorated_function

web/commons/helpers/func.py

@@ -0,0 +1,158 @@
+# import json
+# import time
+# import zlib
+# from datetime import datetime
+# from time import mktime
+#
+# from flask import request, g
+# from nc_http.tools.helpers import strip_value
+#
+#
+# def filter_fields(data, valid_fields):
+#     """
+#     过滤字段
+#     :param data:
+#     :param valid_fields: 格式['field1', 'field2', 'field3', {'field4': ['sub_field1', 'sub_field2']}]
+#     :return:
+#     """
+#     if isinstance(data, list):
+#         tmp = []
+#         for item in data:
+#             tmp.append(filter_fields(item, valid_fields))
+#         format_data = tmp
+#     else:
+#         format_data = {}
+#         for field in valid_fields:
+#             if isinstance(field, dict):
+#                 for key, sub_valid_fields in field.items():
+#                     if isinstance(sub_valid_fields, list) and key in data:
+#                         format_data[key] = filter_fields(data[key], sub_valid_fields)
+#             else:
+#                 if field in data:
+#                     format_data[field] = data[field]
+#     return format_data
+#
+#
+# def get_client_ip(request):
+#     """
+#     获取客户端 ip
+#     :param request:
+#     :return:
+#     """
+#     x_forwarded_for = request.headers.get('X-Forwarded-For')
+#     if x_forwarded_for:
+#         ips = x_forwarded_for.split(',')
+#         return ips[0].strip()
+#     return request.headers.get('X-Real-Ip', request.remote_addr)
+#
+#
+# def get_time(data, is_datetime=False):
+#     if data:
+#         if isinstance(data, int):
+#             return data
+#
+#         if isinstance(data, datetime):
+#             return int(time.mktime(data.timetuple()))
+#
+#         _data = None
+#         if data.isdigit():
+#             _data = int(data)
+#             if is_datetime:
+#                 _data = datetime.fromtimestamp(_data)
+#         else:
+#             if data.count('-') == 2:
+#                 date_format = '%Y-%m-%d'
+#             elif data.count('-') == 1:
+#                 date_format = '%Y-%m'
+#             elif data.count('/') == 2:
+#                 date_format = '%Y/%m/%d'
+#
+#             else:
+#                 date_format = ''
+#
+#             if data.count(':') == 2:
+#                 time_format = '%H:%M:%S'
+#             elif data.count(':') == 1:
+#                 time_format = '%H:%M'
+#             else:
+#                 time_format = ''
+#
+#             if date_format or time_format:
+#                 if ' ' in data:
+#                     datetime_format = '{} {}'.format(date_format, time_format)
+#                 else:
+#                     datetime_format = '{}{}'.format(date_format, time_format)
+#
+#                 _data = datetime.strptime(data, datetime_format)
+#                 if is_datetime is False:
+#                     _data = int(mktime(_data.timetuple()))
+#             else:
+#                 if is_datetime:
+#                     return None
+#                 return 0
+#     else:
+#         if is_datetime:
+#             return None
+#         return 0
+#
+#     return _data
+#
+#
+# def camelize(uncamelized_str):
+#     if not uncamelized_str:
+#         return uncamelized_str
+#     result = ''.join(i.capitalize() for i in uncamelized_str.split('_'))
+#     result = ''.join((result[0].lower(), result[1:]))
+#     return result
+#
+#
+# def uncamelize(camelized_str):
+#     if not camelized_str:
+#         return camelized_str
+#     lst = []
+#     for index, char in enumerate(camelized_str):
+#         if char.isupper() and index != 0:
+#             lst.append("_")
+#         lst.append(char)
+#
+#     return ''.join(lst).lower()
+#
+#
+# def uncamelize_dict(d):
+#     return {uncamelize(k): v for k, v in d.items()}
+#
+#
+# def camelize_dict(d):
+#     return {camelize(k): v for k, v in d.items()}
+#
+#
+# def get_request_json(is_uncamelize=True):
+#     """
+#     获取 json 传递参数
+#     :return:
+#     """
+#     if request.method.lower() == 'get':
+#         data = request.args.to_dict()
+#     else:
+#         if request.content_encoding and 'gzip' in request.content_encoding:
+#             json_data = zlib.decompress(request.get_data())
+#             data = json.loads(json_data)
+#         else:
+#             data = request.get_json(force=True, silent=True) or {}
+#     if is_uncamelize:
+#         data = {uncamelize(k): v for k, v in data.items()}
+#
+#     g.request_data = strip_value(data)
+#
+#     return g.request_data
+#
+#
+# def get_params():
+#     data = request.args.to_dict()
+#     if request.method.lower() != 'get':
+#         if request.content_encoding and 'gzip' in request.content_encoding:
+#             json_data = zlib.decompress(request.get_data())
+#             data.update(json.loads(json_data))
+#         else:
+#             data.update(request.get_json(force=True, silent=True) or {})
+#     return data

web/commons/meta.py

@@ -0,0 +1,42 @@
+import functools
+
+from nc_http.core import ResponseMeta
+
+
+# 请求已成功，请求所希望的响应头或数据体将随此响应返回。
+OK = ResponseMeta(code=200, http_code=200, description='')
+# 请求已经被实现，而且有一个新的资源已经依据请求的需要而创建。
+CREATED = ResponseMeta(code=201, http_code=201, description='')
+# 资源已经删除。
+NO_CONTENT = ResponseMeta(code=204, http_code=204, description='')
+# 由于包含语法错误，当前请求无法被服务器理解。
+BAD_REQUEST = ResponseMeta(code=400, http_code=400, description='请求数据错误!')
+# 当前请求需要用户验证。
+UNAUTHORIZED = ResponseMeta(code=401, http_code=401, description='授权无效!')
+# 服务器已经理解请求，但是拒绝执行它。
+FORBIDDEN = ResponseMeta(code=403, http_code=403, description='没权限访问,请退出重新登录!')
+# 无权限操作
+OPERATE_FORBIDDEN = ResponseMeta(code=407, http_code=403, description='无操作权限')
+# 请求失败，请求所希望得到的资源未被在服务器上发现。
+NOT_FOUND = ResponseMeta(code=404, http_code=404, description='资源不存在!')
+# 请求行中指定的请求方法不能被用于请求相应的资源。
+METHOD_NOT_ALLOWED = ResponseMeta(code=405, http_code=405, description='方法不存在!')
+# 请求的资源的内容特性无法满足请求头中的条件，因而无法生成响应实体。
+NOT_ACCEPTABLE = ResponseMeta(code=406, http_code=406, description='客户端无效!')
+# 服务器遇到了一个未曾预料的状况，导致了它无法完成对请求的处理。
+INTERNAL_SERVER_ERROR = ResponseMeta(code=500, http_code=500, description='服务器内部错误!')
+# 作为网关或者代理工作的服务器尝试执行请求时，从上游服务器接收到无效的响应。
+BAD_GATEWAY = ResponseMeta(code=502, http_code=502, description='接口错误!')
+# 由于临时的服务器维护或者过载，服务器当前无法处理请求。
+SERVICE_UNAVAILABLE = ResponseMeta(code=503, http_code=503, description='服务暂时不可用!')
+# 未能及时从上游服务器收到响应。
+GATEWAY_TIMEOUT = ResponseMeta(code=504, http_code=504, description='接口超时!')
+
+INVALID_PARAMS = functools.partial(lambda **kwargs: ResponseMeta(
+    code=11001, http_code=400,
+    description='非法请求参数{}'.format(': {}'.format(kwargs['hint_message']) if kwargs.get('hint_message') else ''),
+    **kwargs
+))
+
+FILE_NOT_FOUND = ResponseMeta(code=403, http_code=403, description='未获取到文件')
+

web/commons/models/__init__.py

@@ -0,0 +1,176 @@
+from datetime import datetime
+
+import time
+
+from nc_http.tools.helpers import camelize
+from sqlalchemy.orm.exc import NoResultFound
+
+from commons.models.atomic import Atomic
+
+
+class BaseModel:
+    query = None
+
+    _unique_fields = []
+    _db = None
+
+    __table__ = None
+
+    @classmethod
+    def atomic(cls):
+        return Atomic(db=cls._db)
+
+    @classmethod
+    def get_by_id(cls, id_, session=None):
+        if session:
+            query = session.query(cls)
+        else:
+            query = cls.query
+        result = query.filter((getattr(cls, 'id', None) or cls.__table__.primary_key) == id_).one_or_none()
+        return result
+
+    @classmethod
+    def insert_many(cls, dicts, commit=True, session=None):
+        session = session or cls._db.session()
+        session.bulk_insert_mappings(cls, dicts)
+        if commit:
+            session.commit()
+
+    @classmethod
+    def insert_one(cls, values, commit=True, session=None, return_obj=False):
+        session = session or cls._db.session()
+        if not return_obj:
+            session.bulk_insert_mappings(cls, [values])
+        else:
+            obj = cls()
+            for key, value in values.items():
+                setattr(obj, key, value)
+
+            session.add(obj)
+
+        if commit:
+            session.commit()
+
+        if return_obj:
+            return obj
+
+        return
+
+    @classmethod
+    def update_one(cls, values, commit=True, session=None):
+        session = session or cls._db.session()
+        session.bulk_update_mappings(cls, [values])
+        if commit:
+            session.commit()
+
+    @classmethod
+    def update_many(cls, dicts, commit=True, session=None):
+        session = session or cls._db.session()
+        session.bulk_update_mappings(cls, dicts)
+        if commit:
+            session.commit()
+
+    def to_dict(self, is_camelized=True):
+        _ = {}
+        for column in self.__table__.columns:
+            key = getattr(column, 'quote', None) or column.name
+            if is_camelized:
+                _[camelize(key)] = getattr(self, key, None)
+            else:
+                _[key] = getattr(self, key, None)
+        return _
+
+    @classmethod
+    def select_one(cls, query, fields=None):
+        try:
+            result = query.one()
+        except NoResultFound:
+            return {}
+
+        if not result:
+            return {}
+
+        return dict(zip([i.key for i in fields], result))
+
+    @classmethod
+    def test_one(cls):
+        result = cls.query.first()
+        return result.to_dict()
+
+    @classmethod
+    def now(cls):
+        return int(time.time())
+
+    @classmethod
+    def now_datetime(cls):
+        return datetime.fromtimestamp(int(time.time()))
+
+    @classmethod
+    def get_all(cls):
+        objects = cls.query.all()
+        for obj in objects:
+            yield obj.to_dict()
+
+    def update(self, commit=True, session=None, **kwargs):
+        for key, value in kwargs.items():
+            if hasattr(self, key):
+                setattr(self, key, value)
+        if commit:
+            session = session or self._db.session()
+            session.commit()
+
+        return
+
+    def delete(self, session=None):
+        session = session or self._db.session()
+        session.delete(self)
+
+    @classmethod
+    def get_list(cls, query, paging=None, fields=None):
+        if fields:
+            query = query.with_entities(*fields.values())
+
+        if paging:
+            page = query.paginate(paging['page'], paging['limit'])
+            paging['total'] = page.total
+            result = page.items
+        else:
+            result = query.all()
+
+        if fields:
+            columns = fields.keys()
+            result = [dict(zip(columns, _)) for _ in result]
+        else:
+            result = [_.to_dict() for _ in result]
+
+        if paging:
+            return result, paging
+
+        return result
+
+    @classmethod
+    def set_order_by(cls, query, sorting=None):
+        """
+        配置查询结果排序
+        :param query:
+        :param sorting:
+        :return:
+        """
+        if not sorting or not isinstance(sorting, dict):
+            return query
+
+        order_field = sorting.get('order_field')
+        if not order_field:
+            return query
+
+        order = sorting.get('order')
+        if order == 'desc':
+            query = query.order_by(getattr(cls, order_field).desc())
+        else:
+            query = query.order_by(getattr(cls, order_field))
+
+        return query
+
+
+
+

web/commons/models/asphalt_domestic.py

@@ -0,0 +1,50 @@
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint, func
+
+from commons.models.mixin.calculator import CalculatorMixin
+from core.extensions import db
+
+
+class AsphaltDomestic(db.Model, CalculatorMixin):
+    __tablename__ = 'ASPHALT_DOMESTIC'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    date = Column('DATE', Date, comment='日期')
+    from_ = Column('FROM', Date, comment='数据来源')
+
+    __table_args__ = (
+        UniqueConstraint(name, date, from_,  name='Idx_key'),
+        {'comment': '国产沥青'},
+    )
+
+    @classmethod
+    def get_by_key(cls, name, date, from_):
+        query = cls.query
+        query = query.filter(
+            cls.name == name,
+            cls.date == date,
+            cls.from_ == from_,
+        )
+        return query.one_or_none()
+
+    @classmethod
+    def get_items(cls, date,  name_in):
+        query = cls.query
+        query = query.filter(cls.date == date)
+        query = query.filter(cls.name.in_(name_in))
+        query = query.with_entities(cls.name, func.avg(cls.price))
+        query = query.group_by(cls.name)
+        result = query.all()
+        return result
+
+    def upsert(self):
+        result = self.get_by_key(self.name, self.date, self.from_)
+        if not result:
+            session = db.session
+            session.add(self)
+            session.commit()
+        else:
+            session = db.session
+            self.id = result.id
+            session.add(result)
+            session.commit()

web/commons/models/asphalt_imported.py

@@ -0,0 +1,48 @@
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint, func
+
+from commons.models.mixin.calculator import CalculatorMixin
+from core.extensions import db
+
+
+class AsphaltImported(db.Model, CalculatorMixin):
+    __tablename__ = 'ASPHALT_IMPORTED'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, date, name='Idx_key'),
+        {'comment': '进口沥青'},
+    )
+
+    @classmethod
+    def get_items(cls, year, month, name_in):
+        query = cls.query
+        query = cls.query_by_month(query, year, month)
+        query = query.filter(cls.name.in_(name_in))
+        query = query.with_entities(cls.name, func.avg(cls.price))
+        query = query.group_by(cls.name)
+        result = query.all()
+        return result
+
+    @classmethod
+    def get_by_key(cls, name, date):
+        query = cls.query
+        query = query.filter(
+            cls.name == name,
+            cls.date == date,
+        )
+        return query.one_or_none()
+
+    def upsert(self):
+        result = self.get_by_key(self.name, self.date)
+        if not result:
+            session = db.session
+            session.add(self)
+            session.commit()
+        else:
+            session = db.session
+            self.id = result.id
+            session.add(result)
+            session.commit()

web/commons/models/asphalt_modifier.py

@@ -0,0 +1,36 @@
+import datetime
+
+from dateutil.relativedelta import relativedelta
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint
+
+from core.extensions import db
+
+
+class AsphaltModifier(db.Model):
+    __tablename__ = 'ASPHALT_MODIFIER'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, date, name='Idx_key'),
+        {'comment': '沥青改性剂'},
+    )
+
+    @classmethod
+    def get_query(cls, year=None, month=None, name=None, spec=None, name_in=None):
+        query = cls.query
+        if year and month:
+            start_date = datetime.date(year, month, 1)
+            end_date = start_date + relativedelta(months=1)
+            query = query.filter(cls.date >= start_date)
+            query = query.filter(cls.date < end_date)
+        if name:
+            query = query.filter(cls.name == name)
+        if name_in:
+            query = query.filter(cls.name.in_(name_in))
+        if spec:
+            query = query.filter(cls.spec == spec)
+        return query

web/commons/models/atomic/__init__.py

@@ -0,0 +1,23 @@
+from sqlalchemy.orm import sessionmaker
+
+
+class Atomic:
+    def __init__(self, db, session=None):
+        self.session = session or sessionmaker(bind=db.engine)()  # 此处 session 复用会导致事务异常 应创建 session
+
+    def __enter__(self):
+        return self.session
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        if exc_tb is None:
+            try:
+                self.session.commit()
+            except Exception as e:
+                self.session.rollback()
+                raise e
+            finally:
+                self.session.close()
+        else:
+            return False
+
+        return True

web/commons/models/budget.py

@@ -0,0 +1,20 @@
+from sqlalchemy import Column, Integer, String, Date, Text
+
+from core.extensions import db
+
+
+class Budget(db.Model):
+    __tablename__ = 'BUDGET'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    amount = Column('AMOUNT', Integer, default='', comment='总数')
+    min_amount = Column('MIN_AMOUNT', Integer, default='', comment='最小值')
+    max_amount = Column('MAX_AMOUNT', Integer, default='', comment='最大值')
+    year = Column('YEAR', Integer, default=0, comment='年份')
+    months = Column('MONTHS', Text, default='', comment='月份')
+    date = Column('DATE', Date, comment='日期')
+
+    # __table_args__ = (
+    #     UniqueConstraint(name, date, name='Idx_key'),
+    #     {'comment': '预算'},
+    # )

web/commons/models/budget_item.py

@@ -0,0 +1,20 @@
+from sqlalchemy import Column, Integer, String, Text
+
+from core.extensions import db
+
+
+class BudgetItem(db.Model):
+    __tablename__ = 'BUDGET_ITEM'
+    id = Column('ID', Integer, primary_key=True)
+    budget_id = Column('BUDGET_ID', Integer, default='', comment='预算id')
+    name = Column('NAME', String(128), default='', comment='名称')
+    meta = Column('META', Text, default='', comment='数据')
+    quantity = Column('QUANTITY', Integer, default='', comment='数量')
+    unit = Column('UNIT', Text, default='', comment='单位')
+    unit_Price = Column('UNIT_PRICE', Integer, default='', comment='单价')
+    total_Price = Column('TOTAL_PRICE', Integer, default='', comment='总价')
+
+    # __table_args__ = (
+    #     UniqueConstraint(name, date, name='Idx_key'),
+    #     {'comment': '预算细项'},
+    # )

web/commons/models/cement.py

@@ -0,0 +1,59 @@
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint, func
+
+from commons.models.mixin.calculator import CalculatorMixin
+from core.extensions import db
+
+
+class Cement(db.Model, CalculatorMixin):
+    __tablename__ = 'CEMENT'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    pack = Column('PACK', String(64), default='', comment='包装')
+    source = Column('SOURCE', String(64), default='', comment='产地')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    fluctuating = Column('FLUCTUATING', Numeric(16, 4), default=0, comment='浮动')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, pack, source, date, name='Idx_key'),
+        {'comment': '水泥'},
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def get_items(cls, year, month, spec_in, name_in, pack):
+        query = cls.query
+        query = cls.query_by_month(query, year, month)
+        query = query.filter(cls.pack == pack)
+        query = query.filter(cls.spec.in_(spec_in))
+        query = query.filter(cls.name.in_(name_in))
+        query = query.with_entities(func.avg(cls.price))
+        result = query.all()
+        return result
+
+    @classmethod
+    def get_by_key(cls, name, spec, pack, source, date):
+        query = cls.query
+        query = query.filter(
+            cls.name == name,
+            cls.spec == spec,
+            cls.pack == pack,
+            cls.source == source,
+            cls.date == date,
+        )
+        return query.one_or_none()
+
+    def upsert(self):
+        result = self.get_by_key(self.name, self.spec, self.pack, self.source, self.date)
+        if not result:
+            session = db.session
+            session.add(self)
+            session.commit()
+        else:
+            session = db.session
+            self.id = result.id
+            session.add(result)
+            session.commit()

web/commons/models/data_fujian.py

@@ -0,0 +1,40 @@
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint
+
+from commons.models.mixin.base import BaseModelMixin
+from commons.models.model import Model
+from core.extensions import db
+
+
+class DataFujian(db.Model,  Model, BaseModelMixin):
+    __tablename__ = 'DATA_FUJIAN'
+
+    id = Column('ID', Integer, primary_key=True)
+    number = Column('NUMBER', String(128), default='', comment='编码')
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    unit = Column('UNIT', String(128), default='', comment='单位')
+    price_without_tax = Column('PRICE_WITHOUT_TAX', Numeric(16, 4), default=0, comment='除价格')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='含税价')
+    category = Column('CATEGORY', String(128), default='', comment='分类')
+    year = Column('YEAR', Integer, default=0, comment='年份')
+    month = Column('MONTH', Integer, default=0, comment='月份')
+    city = Column('CITY', String(128), default='', comment='地市')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(year, month, city, name, spec, name='Idx_key'),
+        {'comment': '福建数据'},
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def find_by_key(self):
+        query = DataFujian.query
+        query = query.filter(DataFujian.year == self.year)
+        query = query.filter(DataFujian.month == self.month)
+        query = query.filter(DataFujian.city == self.city)
+        query = query.filter(DataFujian.name == self.name)
+        query = query.filter(DataFujian.spec == self.spec)
+        result = query.one_or_none()
+        return result

web/commons/models/data_guangdong.py

@@ -0,0 +1,30 @@
+from sqlalchemy import Column, Integer, String, Date, UniqueConstraint
+
+from commons.models.mixin.base import BaseModelMixin
+from commons.models.model import Model
+from core.extensions import db
+
+
+class DataGuangdong(db.Model,  Model, BaseModelMixin):
+    __tablename__ = 'DATA_GUANGDONG'
+
+    id = Column('ID', Integer, primary_key=True)
+    url = Column('URL', String(512), default='', comment='下载地址')
+    name = Column('NAME', String(128), default='', comment='名称')
+    source = Column('SOURCE', String(128), default='', comment='来源')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, date, name='Idx_key'),
+        {'comment': '广东数据'},
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def find_by_key(self):
+        query = DataGuangdong.query
+        query = query.filter(DataGuangdong.name == self.name)
+        query = query.filter(DataGuangdong.date == self.date)
+        result = query.one_or_none()
+        return result

web/commons/models/data_zhejiang.py

@@ -0,0 +1,30 @@
+from sqlalchemy import Column, Integer, String, Date, UniqueConstraint
+
+from commons.models.mixin.base import BaseModelMixin
+from commons.models.model import Model
+from core.extensions import db
+
+
+class DataZhejiang(db.Model, Model, BaseModelMixin):
+    __tablename__ = 'DATA_ZHEJIANG'
+
+    id = Column('ID', Integer, primary_key=True)
+    url = Column('URL', String(512), default='', comment='下载地址')
+    name = Column('NAME', String(128), default='', comment='名称')
+    source = Column('SOURCE', String(128), default='', comment='来源')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, date, name='Idx_key'),
+        {'comment': '浙江数据'},
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def find_by_key(self):
+        query = DataZhejiang.query
+        query = query.filter(DataZhejiang.name == self.name)
+        query = query.filter(DataZhejiang.date == self.date)
+        result = query.one_or_none()
+        return result

web/commons/models/fujian_survey.py

@@ -0,0 +1,40 @@
+import datetime
+
+from dateutil.relativedelta import relativedelta
+from sqlalchemy import Column, Integer, String, Date, UniqueConstraint, Numeric
+
+from core.extensions import db
+
+
+class FujianSurvey(db.Model):
+    __tablename__ = 'FUJIAN_SURVEY'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    date = Column('DATE', Date, comment='日期')
+    material_id = Column('MATERIAL_ID', String(128), comment='材料id')
+    unit = Column('UNIT', String(128), comment='单位')
+    brand = Column('BRAND', String(128), comment='品牌')
+    tax = Column('TAX', Integer, comment='税率')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, date, name='Idx_key'),
+        {'comment': '福建省交通工程材料调查表'},
+    )
+
+    @classmethod
+    def get_query(cls, year=None, month=None, name=None, spec=None, name_in=None):
+        query = cls.query
+        if year and month:
+            start_date = datetime.date(year, month, 1)
+            end_date = start_date + relativedelta(months=1)
+            query = query.filter(cls.date >= start_date)
+            query = query.filter(cls.date < end_date)
+        if name:
+            query = query.filter(cls.name == name)
+        if name_in:
+            query = query.filter(cls.name.in_(name_in))
+        if spec:
+            query = query.filter(cls.spec == spec)
+        return query

web/commons/models/fuzhou_highway_bureau.py

@@ -0,0 +1,36 @@
+import datetime
+
+from dateutil.relativedelta import relativedelta
+from sqlalchemy import Column, Integer, String, Date, UniqueConstraint, Numeric
+
+from core.extensions import db
+
+
+class FuzhouHighwayBureau(db.Model):
+    __tablename__ = 'FUZHOU_HIGHWAY_BUREAU'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    date = Column('DATE', Date, comment='日期')
+    material_id = Column('MATERIAL_ID', String(128), comment='材料id')
+    unit = Column('UNIT', String(128), comment='单位')
+    brand = Column('BRAND', String(128), comment='品牌')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, date, name='Idx_key'),
+        {'comment': '福州公路局'},
+    )
+
+    @classmethod
+    def get_query(cls, year, month, name, spec=None):
+        start_date = datetime.date(year, month, 1)
+        end_date = start_date + relativedelta(months=1)
+        query = cls.query
+        query = query.filter(cls.date >= start_date)
+        query = query.filter(cls.date < end_date)
+        if name:
+            query = query.filter(cls.name == name)
+        if spec:
+            query = query.filter(cls.spec == spec)
+        return query

web/commons/models/fuzhou_transportation_bureau.py

@@ -0,0 +1,34 @@
+import datetime
+
+from dateutil.relativedelta import relativedelta
+from sqlalchemy import Column, Integer, String, Date, UniqueConstraint, Numeric
+
+from core.extensions import db
+
+
+class FuzhouTransportationBureau(db.Model):
+    __tablename__ = 'FUZHOU_TRANSPORTATION_BUREAU'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    date = Column('DATE', Date, comment='日期')
+    material_id = Column('MATERIAL_ID', String(128), comment='材料id')
+    unit = Column('UNIT', String(128), comment='单位')
+    brand = Column('BRAND', String(128), comment='品牌')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, date, name='Idx_key'),
+        {'comment': '福州交通局'},
+    )
+
+    @classmethod
+    def get_query(cls, year, month, name):
+        start_date = datetime.date(year, month, 1)
+        end_date = start_date + relativedelta(months=1)
+        query = cls.query
+        query = query.filter(cls.date >= start_date)
+        query = query.filter(cls.date < end_date)
+        if name:
+            query = query.filter(cls.name == name)
+        return query

web/commons/models/local_material.py

@@ -0,0 +1,24 @@
+from sqlalchemy import Column, Integer, String, Date, UniqueConstraint, Numeric, Text
+
+from core.extensions import db
+
+
+class LocalMaterial(db.Model):
+    __tablename__ = 'LOCAL_MATERIAL'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    city = Column('CITY', String(128), default='', comment='地市')
+    county = Column('COUNTY', String(128), default='', comment='区县')
+    material_id = Column('MATERIAL_ID', String(128), comment='材料id')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    unit = Column('UNIT', String(128), default='', comment='单位')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    price_without_tax = Column('PRICE_WITHOUT_TAX', Numeric(16, 4), default=0, comment='除税价')
+    date = Column('DATE', Date, comment='日期')
+    position = Column('POSITION', String(256), comment='位置')
+    remark = Column('REMARK', Text, comment='备注')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, city, county, date, name='Idx_key'),
+        {'comment': '地材'},
+    )

web/commons/models/material.py

@@ -0,0 +1,44 @@
+from sqlalchemy import Column, Integer, String
+
+from commons.models.mixin.operation_track import OperationTrackMixin
+from core.extensions import db
+
+
+class Material(db.Model, OperationTrackMixin):
+    id = Column('ID', String(128), primary_key=True)
+    parent_id = Column('PARENT_ID', String(128))
+    category_1 = Column('CATEGORY1', String(128), default='', comment='分类1')
+    category_2 = Column('CATEGORY2', String(128), default='', comment='分类2')
+    category_3 = Column('CATEGORY3', String(128), default='', comment='分类3')
+    category_4 = Column('CATEGORY4', String(128), default='', comment='分类4')
+    name = Column('NAME', String(128), default='', comment='名称')
+    unit = Column('UNIT', String(128), default='', comment='单位')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    tax = Column('TAX', Integer, default=0, comment='税率（%）')
+    is_builtin = Column('IS_BUILTIN', Integer, default=0, comment='是否初始内建类型（不允许删除）')
+    type = Column('TYPE', Integer, default=0, comment='材料类别（主材、地材）')
+    is_tree = Column('IS_TREE', Integer, default=0, comment='是否树')
+
+    __tablename__ = 'MATERIAL'
+    __table_args__ = (
+        {'comment': '材料'},
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def get_by_key(cls, id):
+        return cls.query.filter(cls.id == id).one_or_none()
+
+    def upsert(self):
+        result = self.get_by_key(self.id)
+        if not result:
+            session = db.session
+            session.add(self)
+            session.commit()
+        else:
+            session = db.session
+            self.id = result.id
+            session.add(result)
+            session.commit()

web/commons/models/material_task.py

@@ -0,0 +1,22 @@
+from sqlalchemy import Column, Integer, String
+from sqlalchemy.dialects.mysql import MEDIUMTEXT
+
+from commons.models.mixin.operation_track import OperationTrackMixin
+from commons.models.model import Model
+from core.extensions import db
+
+
+class MaterialTask(db.Model, OperationTrackMixin, Model):
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='任务名称')
+    status = Column('STATUS', Integer, default=0, comment='状态（待采集、已采集、采集中）')
+    file = Column('FILE', String(256), default='', comment='文件路径')
+    type = Column('TYPE', Integer, default=0, comment='类型（网络爬取、文件上传）')
+    year = Column('YEAR', Integer, default=0, comment='采集年份')
+    month = Column('MONTH', Integer, default=0, comment='采集月份')
+    content = Column('CONTENT', MEDIUMTEXT, comment='数据内容')
+
+    __tablename__ = 'MATERIAL_TASK'
+    __table_args__ = (
+        {'comment': '采集任务'},
+    )

web/commons/models/mixin/base.py

@@ -0,0 +1,13 @@
+class BaseModelMixin:
+
+    def find_by_key(self):
+        ...
+
+    def upsert(self):
+        result = self.find_by_key()
+        session = self._db.session
+        if result:
+            session.delete(result)
+            session.flush()
+        session.add(self)
+        session.commit()

web/commons/models/mixin/calculator.py

@@ -0,0 +1,28 @@
+class CalculatorMixin:
+    date = None
+
+    @classmethod
+    def query_by_month(cls, query, year, month):
+        last_month_year, last_month = Helper.get_last_month(year, month)
+        query = query.filter(cls.date >= datetime.date(last_month_year, last_month, 26))
+        query = query.filter(cls.date <= datetime.date(year, month, 25))
+        return query
+
+    def upsert(self, *args, **kwargs):
+        result = self.get_by_key(*args, **kwargs)
+        if not result:
+            session = db.session
+            session.add(self)
+            session.commit()
+        else:
+            session = db.session
+            self.id = result.id
+            session.add(result)
+            session.commit()
+
+    @classmethod
+    def get_last_date_from(cls, date):
+        query = cls.query.filter(cls.date <= date)
+        query = query.with_entities(func.max(cls.date))
+        result = query.one_or_none()
+        return result[0]

web/commons/models/mixin/es_source.py

@@ -0,0 +1,23 @@
+class EsSourceMixin:
+    __table__ = None
+    __tablename__ = ''
+
+    def to_es(self):
+        primary_key_name = [str(i.name) for i in self.__table__.columns if i.primary_key][0]
+        es_data = {
+            'meta': {
+                'id': '{}/{}'.format(self.__tablename__, getattr(self, primary_key_name)),
+            },
+            'table': self.__tablename__,
+            'create_time': self.now() * 1000,
+        }
+        return es_data
+
+    @classmethod
+    def get_es_data(cls):
+        for item in cls.query.yield_per(500):
+            yield item.to_es()
+
+    @classmethod
+    def es_join_str(cls, str_list):
+        return ' '.join(str(i or '') for i in str_list if i)

web/commons/models/mixin/insensitive.py

@@ -0,0 +1,51 @@
+from nc_http.tools.helpers import camelize
+
+
+class InsensitiveMixin:
+    __table__ = None
+    _sensitive_columns = []
+
+    def to_insensitive_dict(self, is_camelized=True):
+        """
+        对象转字典且不带敏感数据 <例：密码、密钥、手机号...>
+        :return:
+        """
+        if self._sensitive_columns:
+            _ = {}
+            for column in self.__table__.columns:
+                if column in self._sensitive_columns:
+                    continue
+                key = getattr(column, 'quote', None) or column.name
+                if is_camelized:
+                    _[camelize(key)] = getattr(self, key, None)
+                else:
+                    _[key] = getattr(self, key, None)
+        else:
+            _ = self.to_dict()
+        return _
+
+    @classmethod
+    def get_list(cls, query, paging=None, fields=None):
+        if fields:
+            query = query.with_entities(*fields.values())
+
+        if paging:
+            page = query.paginate(paging['page'], paging['limit'])
+            paging['total'] = page.total
+            result = page.items
+        else:
+            result = query.all()
+
+        if fields:
+            columns = fields.keys()
+            result = [dict(zip(columns, _)) for _ in result]
+        else:
+            if cls._sensitive_columns:  # 数据脱敏
+                result = [_.to_insensitive_dict() for _ in result]
+            else:
+                result = [_.to_dict() for _ in result]
+
+        if paging:
+            return result, paging
+
+        return result

web/commons/models/mixin/operation_track.py

@@ -0,0 +1,30 @@
+from datetime import datetime
+
+from sqlalchemy import Column, String, DateTime
+
+
+class OperationTrackMixin:
+    update_user_id = Column('update_user_id', String(64), comment='最后更新人id')
+    update_user_name = Column('update_user_name', String(64), comment='最后更新人名称')
+    update_time = Column('update_time', DateTime, default=datetime.now, comment='最后更新时间')
+    create_user_id = Column('create_user_id', String(64), comment='创建人id')
+    create_user_name = Column('create_user_name', String(64), comment='创建人名称')
+    create_time = Column('create_time', DateTime, default=datetime.now, comment='创建时间')
+    delete_user_id = Column('delete_user_id', String(64), comment='删除人id')
+    delete_user_name = Column('delete_user_name', String(64), comment='删除人名称')
+    delete_time = Column('delete_time', DateTime, comment='删除时间')
+
+    def track_delete(self, user_id, user_name=None):
+        self.delete_user_id = user_id
+        self.delete_user_name = user_name
+        self.delete_time = datetime.now()
+
+    def track_create(self, user_id, user_name=None):
+        self.create_user_id = user_id
+        self.create_user_name = user_name
+        self.create_time = datetime.now()
+
+    def track_update(self, user_id, user_name=None):
+        self.update_user_id = user_id
+        self.update_user_name = user_name
+        self.update_time = datetime.now()

web/commons/models/mixin/steel.py

@@ -0,0 +1,25 @@
+class SteelMixin(CalculatorMixin):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def get_by_key(cls, name, spec, material, source, date):
+        spec = spec or ''
+        material = material or ''
+        source = source or ''
+        query = cls.query
+        query = query.filter(cls.name == name).filter(cls.spec == spec).filter(cls.material == material)
+        query = query.filter(cls.source == source).filter(cls.date == date)
+        return query.one_or_none()
+
+    def upsert(self):
+        result = self.get_by_key(self.name, self.spec, self.material, self.source, self.date)
+        if not result:
+            session = db.session
+            session.add(self)
+            session.commit()
+        else:
+            session = db.session
+            self.id = result.id
+            session.add(result)
+            session.commit()

web/commons/models/model.py

@@ -0,0 +1,6 @@
+from commons.models import BaseModel
+from core.extensions import db
+
+
+class Model(BaseModel):
+    _db = db

web/commons/models/oil.py

@@ -0,0 +1,48 @@
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint
+
+from commons.models.mixin.calculator import CalculatorMixin
+from commons.models.model import Model
+from core.extensions import db
+
+
+class Oil(db.Model,  Model, CalculatorMixin):
+    __tablename__ = 'OIL'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, date, name='Idx_key'),
+        {'comment': '成品油'},
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def get_items(cls, year, month, name_in):
+        query = cls.query
+        query = cls.query_by_month(query, year, month)
+        query = query.filter(cls.name.in_(name_in))
+        query = query.order_by(cls.date)
+        result = query.all()
+        return result
+
+    @classmethod
+    def get_by_key(cls, name, date):
+        query = cls.query
+        query = query.filter(cls.name == name).filter(cls.date == date)
+        return query.one_or_none()
+
+    def upsert(self):
+        result = self.get_by_key(self.name, self.date)
+        if not result:
+            session = db.session
+            session.add(self)
+            session.commit()
+        else:
+            session = db.session
+            self.id = result.id
+            session.add(result)
+            session.commit()

web/commons/models/price_publish.py

@@ -0,0 +1,81 @@
+import datetime
+
+from dateutil.relativedelta import relativedelta
+from sqlalchemy import Column, Integer, String, Numeric
+
+from commons.models.mixin.base import BaseModelMixin
+from commons.models.mixin.operation_track import OperationTrackMixin
+from core.extensions import db
+
+
+class PricePublish(db.Model, OperationTrackMixin, BaseModelMixin):
+    id = Column('ID', Integer, primary_key=True)
+    year = Column('YEAR', Integer, default='', comment='统计年份')
+    month = Column('MONTH', Integer, default='', comment='统计月份')
+    material_id = Column('MATERIAL_ID', String(128), default='', comment='编号')
+    name = Column('NAME', String(128), default='', comment='材料名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    price_fuzhou = Column('PRICE_FUZHOU', Numeric(16, 4), default=0, comment='福州价格')
+    price_xiamen = Column('PRICE_XIAMEN', Numeric(16, 4), default=0, comment='厦门价格')
+    price_putian = Column('PRICE_PUTIAN', Numeric(16, 4), default=0, comment='莆田价格')
+    price_sanming = Column('PRICE_SANMING', Numeric(16, 4), default=0, comment='三明价格')
+    price_quanzhou = Column('PRICE_QUANZHOU', Numeric(16, 4), default=0, comment='泉州价格')
+    price_zhangzhou = Column('PRICE_ZHANGZHOU', Numeric(16, 4), default=0, comment='漳州价格')
+    price_nanpin = Column('PRICE_NANPIN', Numeric(16, 4), default=0, comment='南平价格')
+    price_longyan = Column('PRICE_LONGYAN', Numeric(16, 4), default=0, comment='龙岩价格')
+    price_ningde = Column('PRICE_NINGDE', Numeric(16, 4), default=0, comment='宁德价格')
+    price_pintan = Column('PRICE_PINTAN', Numeric(16, 4), default=0, comment='平潭价格')
+    tax = Column('TAX', Numeric(4, 2), default=0, comment='税率')
+    status = Column('STATUS', Integer, default=0, comment='状态')
+    type = Column('TYPE', Integer, default=0, comment='类型')
+    unit = Column('UNIT', String(128), default='', comment='单位')
+
+    __tablename__ = 'PRICE_PUBLISH'
+    __table_args__ = (
+        {'comment': '发布价格'},
+    )
+
+    @classmethod
+    def get_by_key(cls, name, year, month, type):
+        query = cls.query
+        query = query.filter(cls.name == name).filter(cls.year == year).filter(cls.month == month).filter(
+            cls.type == type)
+        return query.one_or_none()
+
+    def find_by_key(self):
+        cls = self.__class__
+        query = cls.query
+        query = query.filter(cls.year == self.year)
+        query = query.filter(cls.month == self.month)
+        query = query.filter(cls.name == self.name)
+        query = query.filter(cls.type == self.type)
+        result = query.one_or_none()
+        return result
+
+    # @classmethod
+    # def query_previous_month(cls, query, start_date: datetime.date, count=6):
+    #     end_date = start_date + relativedelta(months=1)
+    #     query = query.filter(cls.date >= end_date - relativedelta(months=count))
+    #     query = query.filter(cls.date < end_date)
+    #     return query
+
+    @classmethod
+    def query_previous_month(cls, query, start_date: datetime.date, count=6):
+        condition = False
+        for i in range(count):
+            date = start_date - relativedelta(months=i)
+            condition = condition or (cls.year == date.year and cls.month == date.month)
+        query.filter(condition)
+        return query
+
+    @classmethod
+    def get_query(cls, year=None, month=None, name_in=None):
+        query = cls.query
+        if year:
+            query = query.filter(cls.year == year)
+        if month:
+            query = query.filter(cls.month == month)
+        if name_in:
+            query = query.filter(cls.name.in_(name_in))
+        return query

web/commons/models/price_result.py

@@ -0,0 +1,60 @@
+from sqlalchemy import Column, Integer, String, Numeric
+
+from commons.models.mixin.base import BaseModelMixin
+from commons.models.mixin.operation_track import OperationTrackMixin
+from core.extensions import db
+
+
+class PriceResult(db.Model, OperationTrackMixin, BaseModelMixin):
+    id = Column('ID', Integer, primary_key=True)
+    material_id = Column('MATERIAL_ID', String(128), default='', comment='编号')
+    name = Column('NAME', String(128), default='', comment='材料名称')
+    year = Column('YEAR', Integer, default='', comment='统计年份')
+    month = Column('MONTH', Integer, default='', comment='统计月份')
+    price_ftb = Column('PRICE_FTB', Numeric(16, 4), default=0, comment='福州交通局价格')
+    fluctuating_ftb = Column('FLUCTUATING_FTB', Numeric(16, 4), default=0, comment='福州交通局浮动')
+    price_ss = Column('PRICE_SS', Numeric(16, 4), default=0, comment='三明钢铁价格')
+    fluctuating_ss = Column('FLUCTUATING_SS', Numeric(16, 4), default=0, comment='三明钢铁浮动')
+    price_fhb = Column('PRICE_FHB', Numeric(16, 4), default=0, comment='福州公路局价格')
+    fluctuating_fhb = Column('FLUCTUATING_FHB', Numeric(16, 4), default=0, comment='福州公路局浮动')
+    price_network = Column('PRICE_NETWORK', Numeric(16, 4), default=0, comment='网络价格')
+    fluctuating_network = Column('FLUCTUATING_NETWORK', Numeric(16, 4), default=0, comment='网络浮动')
+    price_survey = Column('PRICE_SURVEY', Numeric(16, 4), default=0, comment='调查价格')
+    fluctuating_survey = Column('FLUCTUATING_SURVEY', Numeric(16, 4), default=0, comment='调查浮动')
+    price_last_month = Column('PRICE_LAST_MONTH', Numeric(16, 4), default=0, comment='上月发布价格')
+    price_calculate = Column('PRICE_CALCULATE', Numeric(16, 4), default=0, comment='计算价格')
+    price_recommend = Column('PRICE_RECOMMEND', Numeric(16, 4), default=0, comment='推荐价格')
+    fluctuating_recommend = Column('FLUCTUATING_RECOMMEND', Numeric(16, 4), default=0, comment='推荐浮动')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    unit = Column('UNIT', String(128), default='', comment='单位')
+
+    __tablename__ = 'PRICE_RESULT'
+    __table_args__ = (
+        {'comment': '计算结果'},
+    )
+
+    @classmethod
+    def get_by_key(cls, name, year, month):
+        query = cls.query
+        query = query.filter(cls.name == name).filter(cls.year == year).filter(cls.month == month)
+        return query.one_or_none()
+
+    def find_by_key(self):
+        cls = self.__class__
+        query = cls.query
+        query = query.filter(cls.year == self.year)
+        query = query.filter(cls.month == self.month)
+        query = query.filter(cls.name == self.name)
+        result = query.one_or_none()
+        return result
+
+    @classmethod
+    def get_query(cls, year, month, name_in=None, name=None):
+        query = cls.query
+        query = query.filter(cls.year == year)
+        query = query.filter(cls.month == month)
+        if name_in:
+            query = query.filter(cls.name.in_(name_in))
+        if name:
+            query = query.filter(cls.name == name)
+        return query

web/commons/models/sanming_steel.py

@@ -0,0 +1,35 @@
+import datetime
+
+from dateutil.relativedelta import relativedelta
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint
+
+from core.extensions import db
+
+
+class SanmingSteel(db.Model):
+    __tablename__ = 'SANMING_STEEL'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    material = Column('MATERIAL', String(64), default='', comment='材质')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    fluctuating = Column('FLUCTUATING', Numeric(16, 4), default=0, comment='浮动')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, material, date, name='Idx_key'),
+        {'comment': '三明钢铁'},
+    )
+
+    @classmethod
+    def get_query(cls, year, month, name, spec):
+        start_date = datetime.date(year, month, 1)
+        end_date = start_date + relativedelta(months=1)
+        query = cls.query
+        query = query.filter(cls.date >= start_date)
+        query = query.filter(cls.date < end_date)
+        if name:
+            query = query.filter(cls.name == name)
+        if spec:
+            query = query.filter(cls.spec == spec)
+        return query

web/commons/models/steel_plate.py

@@ -0,0 +1,33 @@
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint, func
+
+from commons.models.mixin.steel import SteelMixin
+from core.extensions import db
+
+
+class SteelPlate(db.Model, SteelMixin):
+    __tablename__ = 'STEEL_PLATE'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    material = Column('MATERIAL', String(64), default='', comment='材质')
+    source = Column('SOURCE', String(64), default='', comment='产地')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    fluctuating = Column('FLUCTUATING', Numeric(16, 4), default=0, comment='浮动')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, material, source, date,  name='Idx_key'),
+        {'comment': '中厚板'},
+    )
+
+    @classmethod
+    def get_items(cls, year, month, spec_in=('12', '16-20', '22-28'), source_in=('三钢闽光',), name_in=('普中板', '工字钢')):
+        query = cls.query
+        query = cls.query_by_month(query, year, month)
+        query = query.filter(cls.spec.in_(spec_in))
+        query = query.filter(cls.source.in_(source_in))
+        query = query.filter(cls.name.in_(name_in))
+        query = query.with_entities(cls.spec, func.avg(cls.price))
+        query = query.group_by(cls.spec)
+        result = query.all()
+        return result

web/commons/models/steel_rebar.py

@@ -0,0 +1,44 @@
+import datetime
+
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint, func
+
+from calculators import Helper
+from commons.models.mixin.steel import SteelMixin
+from core.extensions import db
+
+
+class SteelRebar(db.Model, SteelMixin):
+    __tablename__ = 'STEEL_REBAR'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    material = Column('MATERIAL', String(64), default='', comment='材质')
+    source = Column('SOURCE', String(64), default='', comment='产地')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    fluctuating = Column('FLUCTUATING', Numeric(16, 4), default=0, comment='浮动')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, material, source, date,  name='Idx_key'),
+        {'comment': '钢筋'},
+    )
+
+    @classmethod
+    def query_by_month(cls, query, year, month):
+        last_month_year, last_month = Helper.get_last_month(year, month)
+        query = query.filter(cls.date >= datetime.date(last_month_year, last_month, 26))
+        query = query.filter(cls.date <= datetime.date(year, month, 25))
+        return query
+
+    @classmethod
+    def get_items(cls, year, month, material, source='三钢闽光', name_in=('高线', '螺纹钢')):
+        query = cls.query
+        query = cls.query_by_month(query, year, month)
+        query = query.filter(cls.material == material)
+        query = query.filter(cls.source == source)
+        query = query.filter(cls.name.in_(name_in))
+        query = query.with_entities(cls.spec, cls.material, func.avg(cls.price))
+        query = query.group_by(cls.spec, cls.material)
+        result = query.all()
+
+        return result

web/commons/models/steel_section.py

@@ -0,0 +1,32 @@
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint, func
+
+from commons.models.mixin.steel import SteelMixin
+from core.extensions import db
+
+
+class SteelSection(db.Model, SteelMixin):
+    __tablename__ = 'STEEL_SECTION'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    material = Column('MATERIAL', String(64), default='', comment='材质')
+    source = Column('SOURCE', String(64), default='', comment='产地')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    fluctuating = Column('FLUCTUATING', Numeric(16, 4), default=0, comment='浮动')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, material, source, date,  name='Idx_key'),
+        {'comment': '型钢'},
+    )
+
+    @classmethod
+    def get_items(cls, year, month, spec_in=('140*140*12', '25#'), source_in=('唐山弘泰', '唐山正丰'), name_in=('角钢', '工字钢')):
+        query = cls.query
+        query = cls.query_by_month(query, year, month)
+        query = query.filter(cls.spec.in_(spec_in))
+        query = query.filter(cls.source.in_(source_in))
+        query = query.filter(cls.name.in_(name_in))
+        query = query.with_entities(func.avg(cls.price))
+        result = query.all()
+        return result

web/commons/models/steel_strand.py

@@ -0,0 +1,32 @@
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint, func
+
+from commons.models.mixin.steel import SteelMixin
+from core.extensions import db
+
+
+class SteelStrand(db.Model, SteelMixin):
+    __tablename__ = 'STEEL_STRAND'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    material = Column('MATERIAL', String(64), default='', comment='材质')
+    source = Column('SOURCE', String(64), default='', comment='产地')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    fluctuating = Column('FLUCTUATING', Numeric(16, 4), default=0, comment='浮动')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, material, source, date, name='Idx_key'),
+        {'comment': '钢绞线'},
+    )
+
+    @classmethod
+    def get_items(cls, year, month, material_in=('SWRH82B',), name_in=('弹簧钢',)):
+        query = cls.query
+        query = cls.query_by_month(query, year, month)
+        query = query.filter(cls.material.in_(material_in))
+        query = query.filter(cls.name.in_(name_in))
+        query = query.with_entities(cls.material, func.avg(cls.price))
+        query = query.group_by(cls.material)
+        result = query.all()
+        return result

web/commons/services/data.py

@@ -0,0 +1,50 @@
+import json
+from multiprocessing.pool import Pool
+
+from commons.constants.material_task import MaterialTaskType
+from commons.models.data_fujian import DataFujian
+from commons.models.data_guangdong import DataGuangdong
+from commons.models.data_zhejiang import DataZhejiang
+from commons.models.oil import Oil
+from spiders import run_spider
+from spiders.data_fujian import DataFujianSpider
+from spiders.data_guangdong import DataGuangdongSpider
+from spiders.date_zhejiang import DataZhejiangSpider
+from spiders.oil import OilSpider
+
+
+class DataService:
+    spiders = {
+        MaterialTaskType.OTHER_ZHEJIANG: DataZhejiangSpider,
+        MaterialTaskType.OTHER_GUANGZHOU: DataGuangdongSpider,
+        MaterialTaskType.OTHER_YUNNAN: None,
+        MaterialTaskType.FUJIAN_DEPARTMENT: DataFujianSpider,
+        MaterialTaskType.OIL: OilSpider,
+    }
+    models = {
+        MaterialTaskType.OTHER_ZHEJIANG: DataZhejiang,
+        MaterialTaskType.OTHER_GUANGZHOU: DataGuangdong,
+        MaterialTaskType.OTHER_YUNNAN: None,
+        MaterialTaskType.FUJIAN_DEPARTMENT: DataFujian,
+        MaterialTaskType.OIL: Oil,
+    }
+
+    @classmethod
+    def get_content(cls, type):
+        spider_class = cls.spiders.get(type)
+        if not spider_class:
+            return []
+
+        pool = Pool(processes=1)
+        result = pool.apply_async(run_spider, (spider_class,))
+        pool.close()
+        pool.join()
+        file_path = result.get()
+
+        # file_path = run_spider(spider_class)
+        content = json.loads(open(file_path, 'r', encoding='utf-8').read())
+        model_class = cls.models.get(type)
+        for item in content:
+            model_class(**item).upsert()
+        return content
+

web/commons/vendors/__init__.py

@@ -0,0 +1,66 @@
+from urllib.parse import urlencode
+
+import requests
+
+
+class ApiException(Exception):
+    def __init__(self, code, message):
+        self.code = code
+        self.message = message
+        super().__init__()
+
+    def __str__(self):
+        return self.message
+
+
+class BaseClient:
+    gateway = None
+    timeout = 10
+    max_retries = 3
+
+    def _build_url(self, interface, params, query=None, files=None):
+        method, uri = interface
+
+        query_params = query or {}
+        if params and (method == 'get' or files):
+            query_params.update(params)
+
+        api_url = '{}/{}?{}'.format(self.gateway, uri, urlencode(query_params))
+        return api_url
+
+    def _get_headers(self):
+        return None
+
+    def call(self, interface, params=None, query=None, files=None, max_retries=max_retries, without_unpack=False):
+        method, *_ = interface
+        api_url = self._build_url(interface, params, query, files)
+        headers = self._get_headers()
+        try:
+            if method == 'get':
+                response = requests.request(method, api_url, headers=headers, timeout=self.timeout)
+            else:
+                response = requests.request(
+                    method, api_url, headers=headers, timeout=self.timeout, json=params, files=files
+                )
+        except Exception as e:
+            if max_retries > 0:
+                return self.call(interface, params, files=files, max_retries=max_retries - 1)
+            raise ApiException(504, "GATEWAY TIMEOUT") from e
+
+        if without_unpack:
+            return response
+
+        try:
+            data = self.unpack(response)
+        except ApiException as e:
+            raise e
+        except Exception as e:
+            print('BaseClient.call | interface | {}'.format(interface))
+            print('BaseClient.call | api_url | {}'.format(api_url))
+            raise ApiException(502, "BAD GATEWAY") from e
+
+        return data
+
+    @staticmethod
+    def unpack(response):
+        pass