init

.gitignore

@@ -0,0 +1,38 @@
+venv/
+__pycache__/
+*.ini
+.eggs/
+*.egg-info
+dist/
+build/
+.DS_Store
+*.log
+*.pyc
+.idea
+.sass-cache
+*.iml
+.ipr
+.iws
+*~
+~*
+*.diff
+*.patch
+*.bak
+Thumbs.db
+.project
+.*proj
+.svn
+*.swp
+*.swo
+*.class
+*.tmp_*
+.build
+.logs
+node_modules
+config.rb
+.pytest_cache
+.idea
+
+local_config.py
+.local_config.py
+./local_config.py

.gitlab-ci.yml

@@ -0,0 +1,54 @@
+stages:
+- build
+- deploy
+
+variables:
+  DEPLOY_PATH: '/opt/app/material_api'
+  DEPLOY_DEV_HOST_IP: '192.168.137.123'
+  PIP_CACHE_DIR: "/cache/pip"
+  SSH_OPTS: "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
+  FF_NETWORK_PER_BUILD: "true"
+
+.import_ssh_key: &import_ssh_key
+  before_script:
+  - mkdir -p ~/.ssh
+  - chmod 700 ~/.ssh
+  - echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
+  - echo "${DEPLOY_SSH_KEY}" > ~/.ssh/id_rsa
+  - chmod 600 ~/.ssh/id_rsa
+
+.cache: &cache
+  key: "${CI_PROJECT_PATH}-${CI_COMMIT_REF_SLUG}"
+  paths:
+    - home/
+
+build:
+  stage: build
+  image: python:3.7.16-material-api
+  cache:
+    <<: *cache
+  script:
+    - mkdir dist
+    - PYTHONUSERBASE=$CI_PROJECT_DIR/home pip install --upgrade pip -i http://192.168.137.1:8082/repository/pypi-group/simple --trusted-host 192.168.137.1 --user
+    - PYTHONUSERBASE=$CI_PROJECT_DIR/home pip install -r ./requirements.txt -i http://192.168.137.1:8082/repository/pypi-group/simple --trusted-host 192.168.137.1 --user
+    - rsync -ha ./ ./dist --exclude=dist  --exclude=.git --delete
+  artifacts:
+    paths:
+      - ./dist
+    expire_in: 7 days
+  only:
+    - master
+#    - test
+#    - production
+
+deploy:master:
+  <<: *import_ssh_key
+  image: instrumentisto/rsync-ssh
+  stage: deploy
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - rsync -r ./dist/ ${DEPLOY_DEV_HOST_IP}:${DEPLOY_PATH}  --delete
+    - ssh ${SSH_OPTS} ${DEPLOY_DEV_HOST_IP} "cd ${DEPLOY_PATH} && source /etc/profile && docker-compose up -d --force-recreate"
+  only:
+    - master

README.md

@@ -0,0 +1,2 @@
+挂载SMB文件路径
+mount -o username=administrator,password=Xxs123456Xxs //192.168.137.1/data/fjmap /home/fjmap

docker-compose.yml

@@ -0,0 +1,22 @@
+version: '2.1'
+services:
+  www:
+    image: python:3.7.16-material-api
+    build: ./docker
+    restart: always
+    environment:
+    - SERVICE_NAME=material-api
+    - HOST_IP=${HOST_IP:-127.0.0.1}
+    - TZ="Asia/Shanghai"
+    command:
+      "/root/.local/bin/supervisord -c /usr/src/app/${SUPERVISOR_CONFIG:-supervisor.conf}"
+    extra_hosts:
+    - "test.225.server:${_IP-168.168.7.225}"
+    volumes:
+    - ./web:/usr/src/app
+    - ./home:/root/.local
+    - ./logs:/logs
+    - /opt/files/material_api:/usr/src/app/files
+    working_dir: /usr/src/app/
+    ports:
+    - 7072:7072

init_docker_env.bat

@@ -0,0 +1,4 @@
+docker-compose build
+docker run -it -v "E:\Project\transport_api:/app"  python:3.7.16-transport-api bash
+PYTHONUSERBASE=/app/home pip install -r /app/requirements.txt -i http://192.168.1.3:8082/repository/pypi-group/simple --trusted-host 192.168.1.3 --user
+PYTHONUSERBASE=/app/home pip install nc_http --upgrade -i http://192.168.1.3:8082/repository/pypi-group/simple --trusted-host 192.168.1.3 --user

init_env.bat

@@ -0,0 +1,3 @@
+python -m pip install pip --upgrade -i https://pypi.douban.com/simple
+python -m pip install -r requirements.txt -i https://pypi.douban.com/simple
+python -m pip install nc_http --upgrade -i http://192.168.1.3:8082/repository/pypi-group/simple --trusted-host 192.168.1.3

requirements.txt

@@ -0,0 +1,27 @@
+#git+http://oauth2:4VGjs7rFL9fz8EwR1x6g@192.168.1.3:8929/jgy_xxs/nc_http.git
+#git+http://oauth2:4VGjs7rFL9fz8EwR1x6g@192.168.137.123:8929/jgy_xxs/nc_http.git
+#git+http://oauth2:4VGjs7rFL9fz8EwR1x6g@192.168.1.3:8929/jgy_xxs/nc_http.git
+#git+http://oauth2:4VGjs7rFL9fz8EwR1x6g@gitlab:8929/jgy_xxs/nc_http.git
+#nc_http>=0.0.1.dev54
+
+Flask>=2.0.3
+Flask-SQLAlchemy>=2.4.4
+Flask-APScheduler>=1.11.0
+flask-siwadoc>=0.2.1
+SQLAlchemy>=1.3.23
+requests>=2.25.1
+python-dateutil>=2.8.1
+retrying
+
+psycopg2
+MySQL-connector-python
+
+# deploy
+waitress>=1.4.4
+gevent>=21.1.2
+supervisor>=4.2.1
+gunicorn
+
+scrapy
+Twisted==22.10.0
+selenium

update_home.bat

@@ -0,0 +1,3 @@
+docker run -it -v "E:\Project\geological-map-api:/app"  python:3.6.8-geological-map-api bash
+
+PYTHONUSERBASE=/app/home pip install -r /app/requirements.txt -i http://192.168.1.3:8082/repository/pypi-group/simple --trusted-host 192.168.1.3 --user

update_nc.bat

@@ -0,0 +1 @@
+python37 -m pip install nc_http --upgrade -i http://192.168.1.3:8082/repository/pypi-group/simple --trusted-host 192.168.1.3

web/api/__init__.py

@@ -0,0 +1,99 @@
+
+""" 接口加载器
+"""
+from flask import g, Blueprint
+from nc_http.core import ResponseMeta
+from werkzeug.http import HTTP_STATUS_CODES
+
+import os
+import logging
+from logging.handlers import TimedRotatingFileHandler
+from importlib import import_module
+import glob
+
+from config import APP_MODE, API_ROOT
+
+LOGGER = logging.getLogger()
+
+
+def http_error_handler(err):
+    # 忽略404日志记录
+    if err.code not in [400, 403, 404]:
+        LOGGER.error(err)
+    return ResponseMeta(code=err.code, http_code=err.code, description=err.name)
+
+
+def response_meta_handler(response_meta):
+    return response_meta.get_response()
+
+
+def generic_error_handler(err):
+    LOGGER.exception(err)
+
+    if APP_MODE == 'Production':
+        return ResponseMeta(http_code=500, description='出错了，请及时联系我们，我们的工程师将全力为您解决。')
+    else:
+        return ResponseMeta(http_code=500, description=str(err))
+
+
+def before_request():
+    """
+    g.auth 授权之后
+    g.pagination 返回分页
+    """
+    # 自定义的请求头部
+    g.headers = {}
+
+
+def app_teardown(exc):
+    pass
+
+
+def init_logger(app):
+    log_dir = app.config.get('LOG_DIR')
+
+    if log_dir:
+        file_handler = TimedRotatingFileHandler(os.path.join(log_dir, 'app.log'), 'midnight', 1, 15)
+        file_handler.suffix = '%Y%m%d'
+        file_handler.setFormatter(logging.Formatter(app.config.get('LOG_FORMAT')))
+        file_handler.setLevel(logging.DEBUG)
+        logger = logging.getLogger()
+        logger.addHandler(file_handler)
+        logger.setLevel(logging.DEBUG)
+
+
+# 初始化接口
+def init_app(app):
+    app.teardown_appcontext(app_teardown)
+    app.before_request(before_request)
+
+    for code in HTTP_STATUS_CODES:
+        if code in [400, 401, 403, 404, 405, 406, 409, 500, 502, 503, 504]:
+            app.register_error_handler(code, http_error_handler)
+
+    app.register_error_handler(ResponseMeta, response_meta_handler)
+    app.register_error_handler(Exception, generic_error_handler)
+
+    # from api.helpers.db import main
+    # main()
+
+    # 注册接口
+    path = os.path.dirname(__file__)
+
+    file_names = glob.glob(os.path.join(path, 'components', '*.py'))
+    file_names = file_names + glob.glob(os.path.join(path, 'components', '**', '*.py'))
+    for filename in file_names:
+        filename = filename.replace(os.path.join(path, "components"), '').replace(os.sep, '.')
+        module_name = os.path.basename(filename)[:-3]
+        import_module('.components' + module_name, os.path.basename(path))
+
+    from . import blueprint
+
+    for blueprint_name in dir(blueprint):
+        if not blueprint_name.startswith('__'):
+            bp = getattr(blueprint, blueprint_name)
+            if isinstance(bp, Blueprint):
+                app.register_blueprint(
+                    bp,
+                    url_prefix='{}{}'.format(API_ROOT, bp.url_prefix) if blueprint_name != 'root' else bp.url_prefix
+                )

web/api/blueprint.py

@@ -0,0 +1,5 @@
+from flask import Blueprint
+
+root = Blueprint('root', __name__, url_prefix='/')
+#
+data = Blueprint('data', __name__, url_prefix='/data')

web/api/components/data.py

@@ -0,0 +1,32 @@
+import json
+import traceback
+
+from nc_http.core import Response
+
+from api.blueprint import data
+from commons.constants.material_task import MaterialTaskStatus
+from commons.models.material_task import MaterialTask
+from commons.services.data import DataService
+from core.extensions import siwa
+
+
+@data.route('/task/<int:task_id>/refresh', methods=['GET'])
+@siwa.doc(tags=[""], summary='')
+def refresh_task(task_id):
+    with MaterialTask.atomic() as session:
+        data = MaterialTask.get_by_id(task_id, session)
+        if not data:
+            pass
+        data.status = MaterialTaskStatus.DOING
+        session.flush()
+        try:
+            content = DataService.get_content(type=data.type)
+            data.content = json.dumps(content, ensure_ascii=False)
+            data.status = MaterialTaskStatus.DONE
+            session.flush()
+        except Exception as e:
+            traceback.print_exc()
+            data.status = MaterialTaskStatus.FAILED
+            session.flush()
+
+    return Response()

web/api/components/root.py

@@ -0,0 +1,53 @@
+import json
+import os
+from logging import getLogger
+
+import requests
+from flask import request, send_file
+from nc_http.core import Response
+from nc_http.core.request.func import get_client_ip, get_request_json
+
+import config
+from api.blueprint import root
+from commons import meta
+from tasks import once
+
+LOGGER = getLogger('root')
+
+
+@root.route('')
+def root_check():
+    # 测试服务是否启动
+    return 'ok'
+
+
+@root.route('/ip')
+def get_ip():
+    # 获取客户端 ip
+    ip = get_client_ip(request)
+    return Response(ip)
+
+
+@root.route('/debug/c', methods=['POST'])
+def test_es():
+    data = get_request_json()
+    url = "{}/one-map-data/_search".format(config.ES_HOSTS)
+    headers = {'Content-Type': 'application/json'}
+    result = requests.post(url, json.dumps(data), headers=headers)
+    return Response(json.loads(result.content))
+
+
+@root.route('/debug/run_once/<string:task_name>', methods=['GET'])
+def run_once_task(task_name):
+    if not config.DEBUG:
+        raise meta.NO_CONTENT
+
+    task = getattr(once, task_name)
+
+    return Response(task())
+
+
+@root.route('/logs', methods=['GET'])
+def download_log_file():
+    log_file = os.path.join(config.LOG_DIR, 'app.log')
+    return send_file(log_file, as_attachment=True, download_name='geological-map-api.log', mimetype='text/plain')

web/calculators/__init__.py

@@ -0,0 +1,183 @@
+from commons.models.price_result import PriceResult
+
+
+class Helper:
+
+    @staticmethod
+    def get_last_month(year, month):
+        if month == 1:
+            last_month_year = year - 1
+            last_month = 12
+        else:
+            last_month_year = year
+            last_month = month - 1
+
+        return last_month_year, last_month
+
+    @staticmethod
+    def get_next_month(year, month):
+        if month == 12:
+            last_month_year = year + 1
+            last_month = 1
+        else:
+            last_month_year = year
+            last_month = month + 1
+
+        return last_month_year, last_month
+
+
+class Calculator:
+    material_id = ""
+    name = ""
+    year = 0
+    month = 0
+    price_ftb = 0
+    fluctuating_ftb = 0
+    price_ss = 0
+    fluctuating_ss = 0
+    price_fhb = 0
+    fluctuating_fhb = 0
+    price_network = 0
+    fluctuating_network = 0
+    price_survey = 0
+    fluctuating_survey = 0
+    price_last_month = 0
+    price_calculate = 0
+    price_recommend = 0
+    fluctuating_recommend = 0
+    unit = ""
+    spec = ''
+
+    _previous_prices = None
+    _fluctuatings = []
+
+    @property
+    def previous_prices(self):
+        if not self._previous_prices:
+            previous_price = PriceResult.get_by_key(self.name, *Helper.get_last_month(self.year, self.month))
+            self._previous_prices = previous_price
+        return self._previous_prices
+
+    def result(self):
+        return {
+            'material_id': self.material_id,
+            'name': self.name,
+            'year': self.year,
+            'month': self.month,
+            'price_ftb': self.price_ftb,
+            'fluctuating_ftb': self.fluctuating_ftb,
+            'price_ss': self.price_ss,
+            'fluctuating_ss': self.fluctuating_ss,
+            'price_fhb': self.price_fhb,
+            'fluctuating_fhb': self.fluctuating_fhb,
+            'price_network': self.price_network,
+            'fluctuating_network': self.fluctuating_network,
+            'price_survey': self.price_survey,
+            'fluctuating_survey': self.fluctuating_survey,
+            'price_last_month': self.price_last_month,
+            'price_calculate': self.price_calculate,
+            'price_recommend': self.price_recommend,
+            'fluctuating_recommend': self.fluctuating_recommend,
+            'unit': self.unit,
+            'spec': self.spec,
+        }
+
+    def run(self):
+        self.price_ftb, self.fluctuating_ftb = self._get_ftb_price()
+        self.price_ss, self.fluctuating_ss = self._get_ss_price()
+        self.price_fhb, self.fluctuating_fhb = self._get_fhb_price()
+        self.price_network, self.fluctuating_network = self._get_network_price()
+        self.price_survey, self.fluctuating_survey = self._get_survey_price()
+        self.price_last_month = self._get_last_month_price()
+        self.price_calculate = self._get_calculate_price()
+        self.price_recommend, self.fluctuating_recommend = self._get_recommend_price()
+        return self
+
+    def _get_ftb_price(self):
+        return 0, 0
+
+    def _get_ss_price(self):
+        return 0, 0
+
+    def _get_fhb_price(self):
+        return 0, 0
+
+    def _get_network_price(self):
+        return 0, 0
+
+    def _get_survey_price(self):
+        return 0, 0
+
+    def _get_last_month_price(self):
+        return getattr(self.previous_prices, 'price_recommend', 0)
+
+    def _get_calculate_price(self, round_dit=0):
+        prices = (
+        self.price_ftb, self.price_ss, self.price_fhb, self.price_network, self.price_survey, self.price_last_month)
+        total = sum(float(i) for i in prices)
+        if total == 0:
+            return 0
+        result = total / len([i for i in prices if i != 0])
+        result = round(result, round_dit)
+        return result
+
+    def _get_recommend_price(self, round_by=10):
+        if not self.previous_prices:
+            return 0, 0
+
+        previous_price = int(getattr(self.previous_prices, 'price_recommend', 0))
+        if not previous_price:
+            return 0, 0
+
+        fluctuating = sum(self._fluctuatings) / len(self._fluctuatings)
+        fluctuating = round(fluctuating / round_by) * round_by
+
+        price = fluctuating + previous_price
+        price = round(price / round_by) * round_by
+
+        return price, fluctuating
+
+    def save(self):
+        result = self.result()
+        PriceResult(**result).upsert()
+
+
+if __name__ == '__main__':
+    from calculators.asphalt_domestic import AsphaltDomesticCalculator
+    from calculators.asphalt_imported import AsphaltImportedCalculator
+    from calculators.cement_325 import Cement325Calculator
+    from calculators.cement_425 import Cement425Calculator
+    from calculators.oil_0 import Oil0Calculator
+    from calculators.oil_89 import Oil89Calculator
+    from calculators.oil_92 import Oil92Calculator
+    from calculators.steel_plate import SteelPlateCalculator
+    from calculators.steel_rebar_300 import Reber300Calculator
+    from calculators.steel_rebar_400 import Reber400Calculator
+    from calculators.steel_section import SteelSectionCalculator
+    from calculators.steel_strand import SteelStrandCalculator
+    from calculators.asphalt_domestic_modifier import AsphaltDomesticModifierCalculator
+    from calculators.asphalt_imported_modifier import AsphaltImportedModifierCalculator
+    from core.factory import ClientApp
+
+    for Calculator in [
+        AsphaltDomesticCalculator,
+        AsphaltImportedCalculator,
+        Cement325Calculator,
+        Cement425Calculator,
+        Oil0Calculator,
+        Oil89Calculator,
+        Oil92Calculator,
+        SteelPlateCalculator,
+        Reber300Calculator,
+        Reber400Calculator,
+        SteelSectionCalculator,
+        SteelStrandCalculator,
+        AsphaltDomesticModifierCalculator,
+        AsphaltImportedModifierCalculator,
+    ]:
+        with ClientApp().app_context():
+            for month in range(8, 13):
+                calculator = Calculator(year=2023, month=month)
+                _result = calculator.run()
+                calculator.save()
+                print(_result)

web/calculators/asphalt_domestic.py

@@ -0,0 +1,73 @@
+import datetime
+
+from sqlalchemy import func
+
+from calculators import Calculator, Helper
+from commons.models.fujian_survey import FujianSurvey
+
+from commons.models.asphalt_domestic import AsphaltDomestic
+
+
+class AsphaltDomesticCalculator(Calculator):
+    name = "国产沥青"
+    material_id = "06.60.61.00"
+    unit = "t"
+    spec = ""
+
+    def __init__(self, year, month):
+        self.year = year
+        self.month = month
+
+    def _get_network_price(self):
+        name_in = (
+            '浙江省—镇海炼化(70#,90#,A级)',
+            '福建省—联合石化(70#,A级)',
+            '广东省—茂名石化(70#,90#,A级)',
+            '广东省—中油高富(70#,90#,A级)华南公司',
+        )
+        last_month_year, last_month = Helper.get_last_month(self.year, self.month)
+
+        end_date = AsphaltDomestic.get_last_date_from(date=datetime.date(last_month_year, last_month, 26))
+        start_date = AsphaltDomestic.get_last_date_from(date=datetime.date(self.year, self.month, 25))
+
+        previous_items = AsphaltDomestic.get_items(date=end_date, name_in=name_in)
+        previous_prices = {material: float(price or 0) for material, price in previous_items}
+
+        current_items = AsphaltDomestic.get_items(date=start_date, name_in=name_in)
+        current_prices = {material: float(price or 0) for material, price in current_items}
+
+        fluctuating_a = current_prices.get(name_in[0], 0) - previous_prices.get(name_in[0], 0)
+        fluctuating_b = current_prices.get(name_in[1], 0) - previous_prices.get(name_in[1], 0)
+        fluctuating_c = current_prices.get(name_in[2], 0) - previous_prices.get(name_in[2], 0)
+        fluctuating_d = current_prices.get(name_in[3], 0) - previous_prices.get(name_in[3], 0)
+
+        previous_price = int(getattr(self.previous_prices, 'price_network', 0))
+
+        price = previous_price + 0.4 * fluctuating_a + 0.2 * (fluctuating_b + fluctuating_c + fluctuating_d)
+        fluctuating = price - previous_price
+
+        return price, fluctuating
+
+    def _get_survey_price(self):
+        query = FujianSurvey.get_query(self.year, self.month, name='石油沥青', spec='国产  (路面用)')
+        query = query.with_entities(func.avg(FujianSurvey.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - int(getattr(self.previous_prices, 'price_survey', price))
+        return price, fluctuating
+
+    def _get_recommend_price(self):
+        self._fluctuatings = [self.fluctuating_network, self.fluctuating_survey]
+        return super()._get_recommend_price()
+
+
+if __name__ == '__main__':
+    from core.factory import ClientApp
+
+    with ClientApp().app_context():
+        calculator = AsphaltDomesticCalculator(year=2023, month=12)
+        _result = calculator.run()
+        calculator.save()
+        print(_result)

web/calculators/asphalt_domestic_modifier.py

@@ -0,0 +1,64 @@
+from sqlalchemy import func
+
+from calculators import Calculator, Helper
+from commons.models.asphalt_modifier import AsphaltModifier
+
+from commons.models.price_result import PriceResult
+
+
+class AsphaltDomesticModifierCalculator(Calculator):
+    name = "国产改性沥青"
+    material_id = "06.12.61.00"
+    unit = "t"
+    spec = ""
+
+    def __init__(self, year, month):
+        self.year = year
+        self.month = month
+
+    def _get_recommend_price(self):
+        previous_price = int(getattr(self.previous_prices, 'price_recommend', 0))
+        if not previous_price:
+            return 0, 0
+
+        query = PriceResult.get_query(self.year, self.month, name='国产沥青')
+        result = query.all()
+        if not result:
+            return 0, 0
+
+        raw_data = result[0]
+
+        fluctuating_1 = int(raw_data.fluctuating_recommend)
+
+        query = AsphaltModifier.get_query(self.year, self.month)
+        query = query.with_entities(func.avg(AsphaltModifier.price))
+        result_current = query.one_or_none()
+        if not result_current or not result_current[0]:
+            return 0, 0
+
+        query = AsphaltModifier.get_query(*Helper.get_last_month(self.year, self.month))
+        query = query.with_entities(func.avg(AsphaltModifier.price))
+        result_previous = query.one_or_none()
+        if not result_previous or not result_previous[0]:
+            return 0, 0
+
+        fluctuating_2 = int((result_current[0] - result_previous[0]) * 5 / 100)
+
+        price = previous_price + fluctuating_1 + fluctuating_2
+        price = round(price / 10) * 10
+
+        return price, fluctuating_1 + fluctuating_2
+
+    def save(self):
+        result = self.result()
+        PriceResult(**result).upsert()
+
+
+if __name__ == '__main__':
+    from core.factory import ClientApp
+
+    with ClientApp().app_context():
+        calculator = AsphaltDomesticModifierCalculator(year=2023, month=9)
+        _result = calculator.run()
+        calculator.save()
+        print(_result)

web/calculators/asphalt_imported.py

@@ -0,0 +1,55 @@
+from sqlalchemy import func
+
+from calculators import Calculator
+from commons.models.asphalt_imported import AsphaltImported
+from commons.models.fujian_survey import FujianSurvey
+from commons.models.price_result import PriceResult
+
+
+class AsphaltImportedCalculator(Calculator):
+    name = "进口沥青"
+    material_id = "06.60.60.00"
+    unit = "t"
+    spec = ""
+
+    def __init__(self, year, month):
+        self.year = year
+        self.month = month
+
+    def _get_network_price(self):
+        items = AsphaltImported.get_items(year=self.year, month=self.month, name_in=('新加坡—华东',))
+        prices = {name: float(price or 0) for name, price in items}
+
+        price = prices.get('新加坡—华东', 0)
+        previous_price = int(getattr(self.previous_prices, 'price_network', 0))
+        fluctuating = price - previous_price
+
+        return price, fluctuating
+
+    def _get_survey_price(self):
+        query = FujianSurvey.get_query(self.year, self.month, name='石油沥青', spec='进口  (路面用)')
+        query = query.with_entities(func.avg(FujianSurvey.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - int(getattr(self.previous_prices, 'price_survey', price))
+        return price, fluctuating
+
+    def _get_recommend_price(self):
+        self._fluctuatings = [self.fluctuating_network, self.fluctuating_survey]
+        return super()._get_recommend_price()
+
+    def save(self):
+        result = self.result()
+        PriceResult(**result).upsert()
+
+
+if __name__ == '__main__':
+    from core.factory import ClientApp
+
+    with ClientApp().app_context():
+        calculator = AsphaltImportedCalculator(year=2023, month=10)
+        result = calculator.run()
+        calculator.save()
+        print(result)

web/calculators/asphalt_imported_modifier.py

@@ -0,0 +1,64 @@
+from sqlalchemy import func
+
+from calculators import Calculator, Helper
+from commons.models.asphalt_modifier import AsphaltModifier
+
+from commons.models.price_result import PriceResult
+
+
+class AsphaltImportedModifierCalculator(Calculator):
+    name = "进口改性沥青"
+    material_id = "06.12.60.00"
+    unit = "t"
+    spec = ""
+
+    def __init__(self, year, month):
+        self.year = year
+        self.month = month
+
+    def _get_recommend_price(self):
+        previous_price = int(getattr(self.previous_prices, 'price_recommend', 0))
+        if not previous_price:
+            return 0, 0
+
+        query = PriceResult.get_query(self.year, self.month, name='进口沥青')
+        result = query.all()
+        if not result:
+            return 0, 0
+
+        raw_data = result[0]
+
+        fluctuating_1 = int(raw_data.fluctuating_recommend)
+
+        query = AsphaltModifier.get_query(self.year, self.month)
+        query = query.with_entities(func.avg(AsphaltModifier.price))
+        result_current = query.one_or_none()
+        if not result_current or not result_current[0]:
+            return 0, 0
+
+        query = AsphaltModifier.get_query(*Helper.get_last_month(self.year, self.month))
+        query = query.with_entities(func.avg(AsphaltModifier.price))
+        result_previous = query.one_or_none()
+        if not result_previous or not result_previous[0]:
+            return 0, 0
+
+        fluctuating_2 = int((result_current[0] - result_previous[0]) * 5 / 100)
+
+        price = previous_price + fluctuating_1 + fluctuating_2
+        price = round(price / 10) * 10
+
+        return price, fluctuating_1 + fluctuating_2
+
+    def save(self):
+        result = self.result()
+        PriceResult(**result).upsert()
+
+
+if __name__ == '__main__':
+    from core.factory import ClientApp
+
+    with ClientApp().app_context():
+        calculator = AsphaltImportedModifierCalculator(year=2023, month=9)
+        _result = calculator.run()
+        calculator.save()
+        print(_result)

web/calculators/cement_325.py

@@ -0,0 +1,77 @@
+from sqlalchemy import func
+
+from calculators import Calculator
+from commons.models.cement import Cement
+from commons.models.fujian_survey import FujianSurvey
+from commons.models.fuzhou_highway_bureau import FuzhouHighwayBureau
+from commons.models.fuzhou_transportation_bureau import FuzhouTransportationBureau
+
+from commons.models.price_result import PriceResult
+
+
+class Cement325Calculator(Calculator):
+    name = "32.5 水泥"
+    material_id = "09.60.60.00"
+    unit = "t"
+    spec = ""
+
+    def __init__(self, year, month):
+        self.year = year
+        self.month = month
+
+    def _get_ftb_price(self):
+        query = FuzhouTransportationBureau.get_query(self.year, self.month, name='32.5级水泥')
+        query = query.with_entities(func.avg(FuzhouTransportationBureau.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - int(getattr(self.previous_prices, 'ftb_price', price))
+        return price, fluctuating
+
+    def _get_fhb_price(self):
+        query = FuzhouHighwayBureau.get_query(self.year, self.month, name='32.5级水泥', spec='旋窑（散装）')
+        query = query.with_entities(func.avg(FuzhouHighwayBureau.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - int(getattr(self.previous_prices, 'ftb_price', price))
+        return price, fluctuating
+
+    def _get_network_price(self):
+        result = Cement.get_items(self.year, self.month, spec_in=('台泥', '金牛', '炼石牌'), name_in=('P.S.A32.5', 'M32.5', 'P.P 32.5R'), pack='散装')
+        prices = result[0][0]
+        if not result[0][0]:
+            return 0, 0
+        price = round(prices / 5) * 5
+        fluctuating = price - int(getattr(self.previous_prices, 'price_network', price))
+        return price, fluctuating
+
+    def _get_survey_price(self):
+        query = FujianSurvey.get_query(self.year, self.month, name='32.5级水泥', spec='旋窑')
+        query = query.with_entities(func.avg(FujianSurvey.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - int(getattr(self.previous_prices, 'price_survey', price))
+        return price, fluctuating
+
+    def _get_recommend_price(self, round_by=5):
+        self._fluctuatings = [self.fluctuating_network, self.fluctuating_survey, self.fluctuating_fhb, self.fluctuating_ftb]
+        return super()._get_recommend_price(round_by)
+
+    def save(self):
+        result = self.result()
+        PriceResult(**result).upsert()
+
+
+if __name__ == '__main__':
+    from core.factory import ClientApp
+
+    with ClientApp().app_context():
+        calculator = Cement325Calculator(year=2023, month=12)
+        result = calculator.run()
+        calculator.save()
+        print(result)

web/calculators/cement_425.py

@@ -0,0 +1,81 @@
+from sqlalchemy import func
+
+from calculators import Calculator
+from commons.models.cement import Cement
+from commons.models.fujian_survey import FujianSurvey
+from commons.models.fuzhou_highway_bureau import FuzhouHighwayBureau
+from commons.models.fuzhou_transportation_bureau import FuzhouTransportationBureau
+
+from commons.models.price_result import PriceResult
+
+
+class Cement425Calculator(Calculator):
+    name = "42.5 水泥"
+    material_id = "09.61.60.00"
+    unit = "t"
+    spec = ""
+
+    def __init__(self, year, month):
+        self.year = year
+        self.month = month
+
+    def _get_ftb_price(self):
+        query = FuzhouTransportationBureau.get_query(self.year, self.month, name='42.5级水泥')
+        query = query.with_entities(func.avg(FuzhouTransportationBureau.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+
+        price = int(result[0][0])
+        fluctuating = price - int(getattr(self.previous_prices, 'ftb_price', price))
+        return price, fluctuating
+
+    # def _get_ss_price(self):
+    #     return 4000, -10
+
+    def _get_fhb_price(self):
+        query = FuzhouHighwayBureau.get_query(self.year, self.month, name='42.5级水泥', spec='旋窑（散装）')
+        query = query.with_entities(func.avg(FuzhouHighwayBureau.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - int(getattr(self.previous_prices, 'ftb_price', price))
+        return price, fluctuating
+
+    def _get_network_price(self):
+        result = Cement.get_items(self.year, self.month, spec_in=('台泥', '金牛', '炼石牌'), name_in=('P.O 42.5',), pack='散装')
+        if not result[0][0]:
+            return 0, 0
+        prices = result[0][0]
+        price = round(prices / 5) * 5
+        fluctuating = price - int(getattr(self.previous_prices, 'price_network', price))
+        return price, fluctuating
+
+    def _get_survey_price(self):
+        query = FujianSurvey.get_query(self.year, self.month, name='42.5级水泥', spec='旋窑')
+        query = query.with_entities(func.avg(FujianSurvey.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - int(getattr(self.previous_prices, 'price_survey', price))
+        return price, fluctuating
+
+    def _get_recommend_price(self, round_by=5):
+        self._fluctuatings = [self.fluctuating_network, self.fluctuating_survey, self.fluctuating_fhb, self.fluctuating_ftb]
+        return super()._get_recommend_price(round_by)
+
+    def save(self):
+        result = self.result()
+        PriceResult(**result).upsert()
+
+
+if __name__ == '__main__':
+    from core.factory import ClientApp
+
+    with ClientApp().app_context():
+        calculator = Cement425Calculator(year=2023, month=11)
+        result = calculator.run()
+        calculator.save()
+        print(result)

web/calculators/oil_0.py

@@ -0,0 +1,63 @@
+import datetime
+
+from calculators import Calculator, Helper
+from commons.models.oil import Oil
+
+from commons.models.price_result import PriceResult
+
+
+class Oil0Calculator(Calculator):
+    name = "柴油（0）"
+    material_id = "69.61.60.00"
+    unit = "kg"
+    spec = "0#"
+
+    def __init__(self, year, month):
+        self.year = year
+        self.month = month
+
+    def _get_network_price(self):
+        name_in = ('车用0号柴油（Ⅵ）',)
+        # 获取上月末价格
+        previous_items = Oil.get_items(*Helper.get_last_month(self.year, self.month), name_in=name_in)
+        previous_item = previous_items[-1]
+        previous_price = previous_item.price
+
+        current_items = Oil.get_items(self.year, self.month, name_in=name_in)
+        # 从本月1号开始计算加权总数
+        start_date = datetime.date(self.year, self.month, 1)
+        previous_date = start_date
+        total_with_weight = 0
+        for item in current_items:
+            total_with_weight += previous_price * (item.date - previous_date).days
+            previous_price = item.price
+            previous_date = item.date
+        # 直到月末
+        end_date = datetime.date(*Helper.get_next_month(self.year, self.month), 1) - datetime.timedelta(days=1)
+        if end_date > previous_date:
+            total_with_weight += previous_price * (end_date - previous_date).days
+        # 计算
+        price = round(total_with_weight / (end_date - start_date).days) / 1000
+        fluctuating = price - int(getattr(self.previous_prices, 'price_network', price))
+
+        return round(price, 2), round(fluctuating, 2)
+
+    def _get_calculate_price(self, round_dit=2):
+        return super()._get_calculate_price(round_dit)
+
+    def _get_recommend_price(self):
+        return self.price_network, self.fluctuating_network
+
+    def save(self):
+        result = self.result()
+        PriceResult(**result).upsert()
+
+
+if __name__ == '__main__':
+    from core.factory import ClientApp
+
+    with ClientApp().app_context():
+        calculator = Oil0Calculator(year=2023, month=11)
+        result = calculator.run()
+        calculator.save()
+        print(result)

web/calculators/oil_89.py

@@ -0,0 +1,63 @@
+import datetime
+
+from calculators import Calculator, Helper
+from commons.models.oil import Oil
+
+from commons.models.price_result import PriceResult
+
+
+class Oil89Calculator(Calculator):
+    name = "汽油（89）"
+    material_id = "69.60.60.00"
+    unit = "kg"
+    spec = "89#"
+
+    def __init__(self, year, month):
+        self.year = year
+        self.month = month
+
+    def _get_network_price(self):
+        name_in = ('车用89号汽油（ⅥB）',)
+        # 获取上月末价格
+        previous_items = Oil.get_items(*Helper.get_last_month(self.year, self.month), name_in=name_in)
+        previous_item = previous_items[-1]
+        previous_price = previous_item.price
+
+        current_items = Oil.get_items(self.year, self.month, name_in=name_in)
+        # 从本月1号开始计算加权总数
+        start_date = datetime.date(self.year, self.month, 1)
+        previous_date = start_date
+        total_with_weight = 0
+        for item in current_items:
+            total_with_weight += previous_price * (item.date - previous_date).days
+            previous_price = item.price
+            previous_date = item.date
+        # 直到月末
+        end_date = datetime.date(*Helper.get_next_month(self.year, self.month), 1) - datetime.timedelta(days=1)
+        if end_date > previous_date:
+            total_with_weight += previous_price * (end_date - previous_date).days
+        # 计算
+        price = round(total_with_weight / (end_date - start_date).days) / 1000
+        fluctuating = price - int(getattr(self.previous_prices, 'price_network', price))
+
+        return round(price, 2), round(fluctuating, 2)
+
+    def _get_calculate_price(self, round_dit=2):
+        return super()._get_calculate_price(round_dit)
+
+    def _get_recommend_price(self):
+        return self.price_network, self.fluctuating_network
+
+    def save(self):
+        result = self.result()
+        PriceResult(**result).upsert()
+
+
+if __name__ == '__main__':
+    from core.factory import ClientApp
+
+    with ClientApp().app_context():
+        calculator = Oil89Calculator(year=2023, month=11)
+        result = calculator.run()
+        calculator.save()
+        print(result)

web/calculators/oil_92.py

@@ -0,0 +1,63 @@
+import datetime
+
+from calculators import Calculator, Helper
+from commons.models.oil import Oil
+
+from commons.models.price_result import PriceResult
+
+
+class Oil92Calculator(Calculator):
+    name = "汽油（92）"
+    material_id = "69.60.61.00"
+    unit = "kg"
+    spec = "92#"
+
+    def __init__(self, year, month):
+        self.year = year
+        self.month = month
+
+    def _get_network_price(self):
+        name_in = ('车用92号汽油（ⅥB）',)
+        # 获取上月末价格
+        previous_items = Oil.get_items(*Helper.get_last_month(self.year, self.month), name_in=name_in)
+        previous_item = previous_items[-1]
+        previous_price = previous_item.price
+
+        current_items = Oil.get_items(self.year, self.month, name_in=name_in)
+        # 从本月1号开始计算加权总数
+        start_date = datetime.date(self.year, self.month, 1)
+        previous_date = start_date
+        total_with_weight = 0
+        for item in current_items:
+            total_with_weight += previous_price * (item.date - previous_date).days
+            previous_price = item.price
+            previous_date = item.date
+        # 直到月末
+        end_date = datetime.date(*Helper.get_next_month(self.year, self.month), 1) - datetime.timedelta(days=1)
+        if end_date > previous_date:
+            total_with_weight += previous_price * (end_date - previous_date).days
+        # 计算
+        price = round(total_with_weight / (end_date - start_date).days) / 1000
+        fluctuating = price - int(getattr(self.previous_prices, 'price_network', price))
+
+        return round(price, 2), round(fluctuating, 2)
+
+    def _get_calculate_price(self, round_dit=2):
+        return super()._get_calculate_price(round_dit)
+
+    def _get_recommend_price(self):
+        return self.price_network, self.fluctuating_network
+
+    def save(self):
+        result = self.result()
+        PriceResult(**result).upsert()
+
+
+if __name__ == '__main__':
+    from core.factory import ClientApp
+
+    with ClientApp().app_context():
+        calculator = Oil92Calculator(year=2023, month=11)
+        result = calculator.run()
+        calculator.save()
+        print(result)

web/calculators/steel_plate.py

@@ -0,0 +1,79 @@
+from sqlalchemy import func
+
+from calculators import Calculator
+from commons.models.fujian_survey import FujianSurvey
+from commons.models.fuzhou_highway_bureau import FuzhouHighwayBureau
+from commons.models.fuzhou_transportation_bureau import FuzhouTransportationBureau
+from commons.models.price_result import PriceResult
+from commons.models.steel_plate import SteelPlate
+
+
+class SteelPlateCalculator(Calculator):
+    name = "中厚板"
+    material_id = "17.01.61.00"
+    unit = "t"
+    spec = ""
+
+    def __init__(self, year, month):
+        self.year = year
+        self.month = month
+
+    def _get_ftb_price(self):
+        query = FuzhouTransportationBureau.get_query(self.year, self.month, name='钢板')
+        query = query.with_entities(func.avg(FuzhouTransportationBureau.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - int(getattr(self.previous_prices, 'ftb_price', price))
+        return price, fluctuating
+
+    # def _get_ss_price(self):
+    #     return 4000, -10
+
+    def _get_fhb_price(self):
+        query = FuzhouHighwayBureau.get_query(self.year, self.month, name='钢板')
+        query = query.with_entities(func.avg(FuzhouHighwayBureau.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - int(getattr(self.previous_prices, 'ftb_price', price))
+        return price, fluctuating
+
+    def _get_network_price(self):
+        result = SteelPlate.get_items(self.year, self.month)
+        prices = {spec: float(price) for spec, price in result}
+        price = 0.2 * prices.get('12', 0) + 0.6 * prices.get('16-20', 0) + 0.2 * prices.get('22-28', 0)
+        price = round(price)
+
+        fluctuating = price - getattr(self.previous_prices, 'price_network', price)
+        return price, fluctuating
+
+    def _get_survey_price(self):
+        query = FujianSurvey.get_query(self.year, self.month, name='钢板')
+        query = query.with_entities(func.avg(FujianSurvey.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - getattr(self.previous_prices, 'price_survey', price)
+        return price, fluctuating
+
+    def _get_recommend_price(self):
+        self._fluctuatings = [self.fluctuating_network, self.fluctuating_survey, self.fluctuating_fhb, self.fluctuating_ftb]
+        return super()._get_recommend_price()
+
+    def save(self):
+        result = self.result()
+        PriceResult(**result).upsert()
+
+
+if __name__ == '__main__':
+    from core.factory import ClientApp
+
+    with ClientApp().app_context():
+        calculator = SteelPlateCalculator(year=2023, month=11)
+        result = calculator.run()
+        calculator.save()
+        print(result)

web/calculators/steel_rebar_300.py

@@ -0,0 +1,88 @@
+from sqlalchemy import func
+
+from calculators import Calculator
+from commons.models.fujian_survey import FujianSurvey
+from commons.models.fuzhou_highway_bureau import FuzhouHighwayBureau
+from commons.models.fuzhou_transportation_bureau import FuzhouTransportationBureau
+from commons.models.price_result import PriceResult
+from commons.models.sanming_steel import SanmingSteel
+from commons.models.steel_rebar import SteelRebar
+
+
+class Reber300Calculator(Calculator):
+    name = "光圆钢筋"
+    material_id = "15.05.60.00"
+    unit = "t"
+    spec = ""
+
+    def __init__(self, year, month):
+        self.year = year
+        self.month = month
+
+    def _get_ftb_price(self):
+        query = FuzhouTransportationBureau.get_query(self.year, self.month, name='光圆钢筋')
+        query = query.with_entities(func.avg(FuzhouTransportationBureau.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - getattr(self.previous_prices, 'ftb_price', price)
+        return price, fluctuating
+
+    def _get_ss_price(self):
+        query = SanmingSteel.get_query(self.year, self.month, name='高线', spec='φ10mm')
+        query = query.with_entities(func.avg(SanmingSteel.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - getattr(self.previous_prices, 'ss_price', price)
+        return price, fluctuating
+
+    def _get_fhb_price(self):
+        query = FuzhouHighwayBureau.get_query(self.year, self.month, name='光圆钢筋')
+        query = query.with_entities(func.avg(FuzhouHighwayBureau.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - getattr(self.previous_prices, 'ftb_price', price)
+        return price, fluctuating
+
+    def _get_network_price(self):
+        result = SteelRebar.get_items(self.year, self.month, 'HPB300')
+        price_dict = {spec: price for spec, material, price in result}
+        price = 0.3 * float(price_dict.get('Φ6', 0)) + 0.7 * float(price_dict.get('Φ8-10', 0))
+        price = round(price)
+
+        fluctuating = price - getattr(self.previous_prices, 'price_network', price)
+        return price, fluctuating
+
+    def _get_survey_price(self):
+        query = FujianSurvey.get_query(self.year, self.month, name='光圆钢筋')
+        query = query.with_entities(func.avg(FujianSurvey.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - getattr(self.previous_prices, 'price_survey', price)
+        return price, fluctuating
+
+    def _get_recommend_price(self):
+        self._fluctuatings = [self.fluctuating_network, self.fluctuating_survey, self.fluctuating_fhb, self.fluctuating_ftb, self.fluctuating_ss]
+        return super()._get_recommend_price()
+
+    def save(self):
+        result = self.result()
+        PriceResult(**result).upsert()
+
+
+if __name__ == '__main__':
+    from core.factory import ClientApp
+
+    with ClientApp().app_context():
+        calculator = Reber300Calculator(year=2023, month=10)
+        result = calculator.run()
+        calculator.save()
+        print(result)
+

web/calculators/steel_rebar_400.py

@@ -0,0 +1,90 @@
+from sqlalchemy import func
+
+from calculators import Calculator
+from commons.models.fujian_survey import FujianSurvey
+from commons.models.fuzhou_highway_bureau import FuzhouHighwayBureau
+from commons.models.fuzhou_transportation_bureau import FuzhouTransportationBureau
+from commons.models.price_result import PriceResult
+from commons.models.sanming_steel import SanmingSteel
+from commons.models.steel_rebar import SteelRebar
+
+
+class Reber400Calculator(Calculator):
+    name = "带肋钢筋"
+    material_id = "15.03.60.00"
+    unit = "t"
+    spec = ""
+
+    # todo price result 加 year 字段
+
+    def __init__(self, year, month):
+        self.year = year
+        self.month = month
+
+    def _get_ftb_price(self):
+        query = FuzhouTransportationBureau.get_query(self.year, self.month, name='带肋钢筋')
+        query = query.with_entities(func.avg(FuzhouTransportationBureau.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - getattr(self.previous_prices, 'ftb_price', price)
+        return price, fluctuating
+
+    def _get_ss_price(self):
+        query = SanmingSteel.get_query(self.year, self.month, name='Ⅲ级螺纹钢筋', spec='HRB400Ф16-25')
+        query = query.with_entities(func.avg(SanmingSteel.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - getattr(self.previous_prices, 'ss_price', price)
+        return price, fluctuating
+
+    def _get_fhb_price(self):
+        query = FuzhouHighwayBureau.get_query(self.year, self.month, name='带肋钢筋')
+        query = query.with_entities(func.avg(FuzhouHighwayBureau.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - getattr(self.previous_prices, 'ftb_price', price)
+        return price, fluctuating
+
+    def _get_network_price(self):
+        result = SteelRebar.get_items(self.year, self.month, 'HRB400E')
+        price_dict = {spec: price for spec, material, price in result}
+        price = 0.2 * float(price_dict.get('Φ12', 0)) + 0.2 * float(price_dict.get('Φ14', 0)) + \
+                0.4 * float(price_dict.get('Φ18-22', 0)) + 0.2 * float(price_dict.get('Φ28-32', 0))
+        price = round(price)
+
+        fluctuating = price - getattr(self.previous_prices, 'price_network', price)
+        return price, fluctuating
+
+    def _get_survey_price(self):
+        query = FujianSurvey.get_query(self.year, self.month, name='带肋钢筋')
+        query = query.with_entities(func.avg(FujianSurvey.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - getattr(self.previous_prices, 'price_survey', price)
+        return price, fluctuating
+
+    def _get_recommend_price(self):
+        self._fluctuatings = [self.fluctuating_network, self.fluctuating_survey, self.fluctuating_fhb, self.fluctuating_ftb, self.fluctuating_ss]
+        return super()._get_recommend_price()
+
+    def save(self):
+        result = self.result()
+        PriceResult(**result).upsert()
+
+
+if __name__ == '__main__':
+    from core.factory import ClientApp
+
+    with ClientApp().app_context():
+        calculator = Reber400Calculator(year=2023, month=10)
+        result = calculator.run()
+        calculator.save()
+        print(result)

web/calculators/steel_section.py

@@ -0,0 +1,79 @@
+from sqlalchemy import func
+
+from calculators import Calculator
+from commons.models.fujian_survey import FujianSurvey
+from commons.models.fuzhou_highway_bureau import FuzhouHighwayBureau
+from commons.models.fuzhou_transportation_bureau import FuzhouTransportationBureau
+from commons.models.price_result import PriceResult
+from commons.models.steel_section import SteelSection
+
+
+class SteelSectionCalculator(Calculator):
+    name = "型钢"
+    material_id = "16.05.60.00"
+    unit = "t"
+    spec = ""
+
+    def __init__(self, year, month):
+        self.year = year
+        self.month = month
+
+    def _get_ftb_price(self):
+        query = FuzhouTransportationBureau.get_query(self.year, self.month, name='型钢')
+        query = query.with_entities(func.avg(FuzhouTransportationBureau.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - getattr(self.previous_prices, 'ftb_price', price)
+        return price, fluctuating
+
+    # def _get_ss_price(self):
+    #     return 4000, -10
+
+    def _get_fhb_price(self):
+        query = FuzhouHighwayBureau.get_query(self.year, self.month, name='型钢')
+        query = query.with_entities(func.avg(FuzhouHighwayBureau.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - getattr(self.previous_prices, 'ftb_price', price)
+        return price, fluctuating
+
+    def _get_network_price(self):
+        result = SteelSection.get_items(self.year, self.month)
+        if not result[0][0]:
+            return 0, 0
+        price = round(result[0][0])
+
+        fluctuating = price - getattr(self.previous_prices, 'price_network', price)
+        return price, fluctuating
+
+    def _get_survey_price(self):
+        query = FujianSurvey.get_query(self.year, self.month, name='型钢')
+        query = query.with_entities(func.avg(FujianSurvey.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - getattr(self.previous_prices, 'price_survey', price)
+        return price, fluctuating
+
+    def _get_recommend_price(self):
+        self._fluctuatings = [self.fluctuating_network, self.fluctuating_survey, self.fluctuating_fhb, self.fluctuating_ftb]
+        return super()._get_recommend_price()
+
+    def save(self):
+        result = self.result()
+        PriceResult(**result).upsert()
+
+
+if __name__ == '__main__':
+    from core.factory import ClientApp
+
+    with ClientApp().app_context():
+        calculator = SteelSectionCalculator(year=2023, month=11)
+        result = calculator.run()
+        calculator.save()
+        print(result)

web/calculators/steel_strand.py

@@ -0,0 +1,78 @@
+from sqlalchemy import func
+
+from calculators import Calculator
+from commons.models.fujian_survey import FujianSurvey
+from commons.models.fuzhou_highway_bureau import FuzhouHighwayBureau
+from commons.models.fuzhou_transportation_bureau import FuzhouTransportationBureau
+from commons.models.price_result import PriceResult
+from commons.models.steel_strand import SteelStrand
+
+
+class SteelStrandCalculator(Calculator):
+    name = "钢绞线"
+    material_id = "19.04.60.00"
+    unit = "t"
+    spec = ""
+
+    def __init__(self, year, month):
+        self.year = year
+        self.month = month
+
+    def _get_ftb_price(self):
+        query = FuzhouTransportationBureau.get_query(self.year, self.month, name='钢绞线')
+        query = query.with_entities(func.avg(FuzhouTransportationBureau.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - getattr(self.previous_prices, 'ftb_price', price)
+        return price, fluctuating
+
+    # def _get_ss_price(self):
+    #     return 4000, -10
+
+    def _get_fhb_price(self):
+        query = FuzhouHighwayBureau.get_query(self.year, self.month, name='钢绞线')
+        query = query.with_entities(func.avg(FuzhouHighwayBureau.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - getattr(self.previous_prices, 'ftb_price', price)
+        return price, fluctuating
+
+    def _get_network_price(self):
+        result = SteelStrand.get_items(self.year, self.month)
+        prices = {material: float(price) for material, price in result}
+        price = round(prices.get('SWRH82B', 0))
+
+        fluctuating = price - getattr(self.previous_prices, 'price_network', price)
+        return price, fluctuating
+
+    def _get_survey_price(self):
+        query = FujianSurvey.get_query(self.year, self.month, name='钢绞线')
+        query = query.with_entities(func.avg(FujianSurvey.price))
+        result = query.all()
+        if not result[0][0]:
+            return 0, 0
+        price = int(result[0][0])
+        fluctuating = price - getattr(self.previous_prices, 'price_survey', price)
+        return price, fluctuating
+
+    def _get_recommend_price(self):
+        self._fluctuatings = [self.fluctuating_network, self.fluctuating_survey, self.fluctuating_fhb, self.fluctuating_ftb]
+        return super()._get_recommend_price()
+
+    def save(self):
+        result = self.result()
+        PriceResult(**result).upsert()
+
+
+if __name__ == '__main__':
+    from core.factory import ClientApp
+
+    with ClientApp().app_context():
+        calculator = SteelStrandCalculator(year=2023, month=11)
+        result = calculator.run()
+        calculator.save()
+        print(result)

web/collectors/__init__.py

@@ -0,0 +1,277 @@
+import datetime
+
+from sqlalchemy import func
+
+from commons.models.fujian_survey import FujianSurvey
+from commons.models.price_publish import PricePublish
+from commons.models.price_result import PriceResult
+
+
+class Collector:
+
+    def __init__(self, year, month):
+        self.year = year
+        self.month = month
+
+    def get_avg(self):
+        query = PricePublish.get_query(name_in=(
+            '杉原木',
+            '松原木',
+            '锯材',
+            '锯材',
+            '毛竹',
+            '冷轧带肋钢筋网',
+            '预应力粗钢筋',
+            '高强钢丝',
+            '钢板',
+            '钢板',
+            '圆钢',
+            '钢轨',
+            '钢管',
+            '镀锌钢管',
+            '镀锌无缝钢管',
+            '镀锌钢板',
+            '钢丝绳',
+            '波形钢板(双波)',
+            '波形钢板(三波)',
+            '四氟板式橡胶组合支座',
+            '板式橡胶支座',
+            '盆式橡胶支座',
+            '盆式橡胶支座',
+            '盆式橡胶支座',
+            '盆式橡胶支座',
+            '盆式橡胶支座',
+            '盆式橡胶支座',
+            '模数式伸缩装置',
+            '模数式伸缩装置',
+            '钢绞线群锚(3孔)',
+            '钢绞线群锚(7孔)',
+            '钢绞线群锚(15孔)',
+            '不锈钢板',
+            '铁件',
+            '铝合金标志',
+            '铸铁管',
+            '钢板网',
+            '铁丝编织网',
+            '橡胶护舷',
+            '橡胶护舷',
+            '橡胶护舷',
+            '鼓型橡胶护舷',
+            '鼓型橡胶护舷',
+            '鼓型橡胶护舷',
+            '油漆',
+            '塑料排水板',
+            '路缘石',
+            '乳化炸药',
+            '硝铵炸药',
+            '导火线',
+            '普通雷管',
+            '电雷管',
+            '工业数码电子雷管',
+            '导爆索',
+            '引爆母线',
+            '重油',
+
+            '光圆钢筋',
+            '带肋钢筋',
+            '型钢',
+            '钢绞线',
+            '32.5级水泥',
+            '42.5级水泥',
+            '进口沥青',
+            '国产沥青',
+            '进口改性沥青',
+            '国产改性沥青',
+            '汽油（89）',
+            '汽油（92）',
+            '柴油（0）',
+        ))
+        query = PricePublish.query_previous_month(query, start_date=datetime.date(self.year, self.month, 1), count=6)
+        query = query.with_entities(
+            PricePublish.material_id,
+            PricePublish.name,
+            PricePublish.spec,
+            func.avg(PricePublish.price),
+            func.avg(PricePublish.price_fuzhou),
+            func.avg(PricePublish.price_xiamen),
+            func.avg(PricePublish.price_putian),
+            func.avg(PricePublish.price_sanming),
+            func.avg(PricePublish.price_quanzhou),
+            func.avg(PricePublish.price_zhangzhou),
+            func.avg(PricePublish.price_nanpin),
+            func.avg(PricePublish.price_longyan),
+            func.avg(PricePublish.price_ningde),
+            func.avg(PricePublish.price_pintan),
+            PricePublish.tax,
+            PricePublish.unit,
+        )
+        query = query.group_by(
+            PricePublish.material_id,
+            PricePublish.name,
+            PricePublish.spec,
+            PricePublish.tax,
+            PricePublish.unit,
+        )
+        data = query.all()
+        for item in data:
+            material_id, name, spec, price, price_fuzhou, price_xiamen, price_putian, price_sanming, price_quanzhou, \
+            price_zhangzhou, price_nanpin, price_longyan, price_ningde, price_pintan, tax, unit = item
+            PricePublish(
+                year=self.year,
+                month=self.month,
+                material_id=material_id,
+                name=name,
+                spec=spec,
+                price=price,
+                price_fuzhou=price_fuzhou,
+                price_xiamen=price_xiamen,
+                price_putian=price_putian,
+                price_sanming=price_sanming,
+                price_quanzhou=price_quanzhou,
+                price_zhangzhou=price_zhangzhou,
+                price_nanpin=price_nanpin,
+                price_longyan=price_longyan,
+                price_ningde=price_ningde,
+                price_pintan=price_pintan,
+                tax=tax,
+                type=2,
+                unit=unit,
+            ).upsert()
+
+    def get_from_survey(self):
+        query = FujianSurvey.get_query(self.year, self.month, name_in=(
+            '杉原木',
+            '松原木',
+            '锯材',
+            '锯材',
+            '毛竹',
+            '冷轧带肋钢筋网',
+            '预应力粗钢筋',
+            '高强钢丝',
+            '钢板',
+            '钢板',
+            '圆钢',
+            '钢轨',
+            '钢管',
+            '镀锌钢管',
+            '镀锌无缝钢管',
+            '镀锌钢板',
+            '钢丝绳',
+            '波形钢板(双波)',
+            '波形钢板(三波)',
+            '四氟板式橡胶组合支座',
+            '板式橡胶支座',
+            '盆式橡胶支座',
+            '盆式橡胶支座',
+            '盆式橡胶支座',
+            '盆式橡胶支座',
+            '盆式橡胶支座',
+            '盆式橡胶支座',
+            '模数式伸缩装置',
+            '模数式伸缩装置',
+            '钢绞线群锚(3孔)',
+            '钢绞线群锚(7孔)',
+            '钢绞线群锚(15孔)',
+            '不锈钢板',
+            '铁件',
+            '铝合金标志',
+            '铸铁管',
+            '钢板网',
+            '铁丝编织网',
+            '橡胶护舷',
+            '橡胶护舷',
+            '橡胶护舷',
+            '鼓型橡胶护舷',
+            '鼓型橡胶护舷',
+            '鼓型橡胶护舷',
+            '油漆',
+            '塑料排水板',
+            '路缘石',
+            '乳化炸药',
+            '硝铵炸药',
+            '导火线',
+            '普通雷管',
+            '电雷管',
+            '工业数码电子雷管',
+            '导爆索',
+            '引爆母线',
+            '重油',
+        ))
+        data = query.all()
+        for item in data:
+            PricePublish(
+                year=self.year,
+                month=self.month,
+                material_id=item.material_id,
+                name=item.name,
+                spec=item.spec,
+                price=item.price,
+                price_fuzhou=item.price,
+                price_xiamen=item.price,
+                price_putian=item.price,
+                price_sanming=item.price,
+                price_quanzhou=item.price,
+                price_zhangzhou=item.price,
+                price_nanpin=item.price,
+                price_longyan=item.price,
+                price_ningde=item.price,
+                price_pintan=item.price,
+                tax=item.tax,
+                type=1,
+                unit=item.unit
+            ).upsert()
+
+    def get_from_result(self):
+        query = PriceResult.get_query(self.year, self.month, name_in=(
+            '光圆钢筋',
+            '带肋钢筋',
+            '型钢',
+            '钢绞线',
+            '32.5级水泥',
+            '42.5级水泥',
+            '进口沥青',
+            '国产沥青',
+            '进口改性沥青',
+            '国产改性沥青',
+            '汽油（89）',
+            '汽油（92）',
+            '柴油（0）',
+        ))
+        data = query.all()
+        for item in data:
+            PricePublish(
+                year=self.year,
+                month=self.month,
+                material_id=item.material_id,
+                name=item.name,
+                spec='',
+                price=item.price_recommend,
+                price_fuzhou=item.price_recommend,
+                price_xiamen=item.price_recommend,
+                price_putian=item.price_recommend,
+                price_sanming=item.price_recommend,
+                price_quanzhou=item.price_recommend,
+                price_zhangzhou=item.price_recommend,
+                price_nanpin=item.price_recommend,
+                price_longyan=item.price_recommend,
+                price_ningde=item.price_recommend,
+                price_pintan=item.price_recommend,
+                tax=9.00,
+                type=1,
+            ).upsert()
+
+    def run(self):
+        self.get_from_survey()
+        self.get_from_result()
+        self.get_avg()
+
+
+# todo 调查表入库时名称去除空格
+
+
+if __name__ == '__main__':
+    from core.factory import ClientApp
+
+    with ClientApp().app_context():
+        collector = Collector(2023, 11)
+        collector.run()

web/commons/constants/__init__.py

@@ -0,0 +1 @@
+from . import *

web/commons/constants/material_task.py

@@ -0,0 +1,21 @@
+class MaterialTaskType:
+    OTHER_ZHEJIANG = 601  # 浙江
+    OTHER_GUANGZHOU = 602  # 广州
+    OTHER_YUNNAN = 603  # 云南
+    FUJIAN_DEPARTMENT = 701  # 住建厅
+    OIL = 801  # 汽柴油
+
+    values = {
+        OTHER_ZHEJIANG: '浙江',
+        OTHER_GUANGZHOU: '广州',
+        OTHER_YUNNAN: '云南',
+        FUJIAN_DEPARTMENT: '住建厅',
+        OIL: '汽柴油',
+    }
+
+
+class MaterialTaskStatus:
+    WAITING = 0
+    DOING = 1
+    DONE = 2
+    FAILED = -1

web/commons/constants/mysteel.py

@@ -0,0 +1,16 @@
+
+class PageType:
+    REBAR_LIST = '福州建筑钢材价格行情'
+    REBAR_DETAIL = '福州建筑钢材价格行情详情'
+    PLATE_LIST = '福州中板价格行情'
+    PLATE_DETAIL = '福州中板价格行情详情'
+    SECTION_LIST = '福州型钢价格行情'
+    SECTION_DETAIL = '福州型钢价格行情详情'
+    SHAGANG_LIST = '沙钢调价信息'
+    SHAGANG_DETAIL = '沙钢调价信息详情'
+    SANGANG_LIST = '三钢调价信息'
+    SANGANG_DETAIL = '三钢调价信息详情'
+
+
+
+

web/commons/helpers/auth.py

@@ -0,0 +1,9 @@
+from functools import wraps
+
+
+def basic_auth_required(f):
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        return f(*args, **kwargs)
+
+    return decorated_function

web/commons/helpers/func.py

@@ -0,0 +1,158 @@
+# import json
+# import time
+# import zlib
+# from datetime import datetime
+# from time import mktime
+#
+# from flask import request, g
+# from nc_http.tools.helpers import strip_value
+#
+#
+# def filter_fields(data, valid_fields):
+#     """
+#     过滤字段
+#     :param data:
+#     :param valid_fields: 格式['field1', 'field2', 'field3', {'field4': ['sub_field1', 'sub_field2']}]
+#     :return:
+#     """
+#     if isinstance(data, list):
+#         tmp = []
+#         for item in data:
+#             tmp.append(filter_fields(item, valid_fields))
+#         format_data = tmp
+#     else:
+#         format_data = {}
+#         for field in valid_fields:
+#             if isinstance(field, dict):
+#                 for key, sub_valid_fields in field.items():
+#                     if isinstance(sub_valid_fields, list) and key in data:
+#                         format_data[key] = filter_fields(data[key], sub_valid_fields)
+#             else:
+#                 if field in data:
+#                     format_data[field] = data[field]
+#     return format_data
+#
+#
+# def get_client_ip(request):
+#     """
+#     获取客户端 ip
+#     :param request:
+#     :return:
+#     """
+#     x_forwarded_for = request.headers.get('X-Forwarded-For')
+#     if x_forwarded_for:
+#         ips = x_forwarded_for.split(',')
+#         return ips[0].strip()
+#     return request.headers.get('X-Real-Ip', request.remote_addr)
+#
+#
+# def get_time(data, is_datetime=False):
+#     if data:
+#         if isinstance(data, int):
+#             return data
+#
+#         if isinstance(data, datetime):
+#             return int(time.mktime(data.timetuple()))
+#
+#         _data = None
+#         if data.isdigit():
+#             _data = int(data)
+#             if is_datetime:
+#                 _data = datetime.fromtimestamp(_data)
+#         else:
+#             if data.count('-') == 2:
+#                 date_format = '%Y-%m-%d'
+#             elif data.count('-') == 1:
+#                 date_format = '%Y-%m'
+#             elif data.count('/') == 2:
+#                 date_format = '%Y/%m/%d'
+#
+#             else:
+#                 date_format = ''
+#
+#             if data.count(':') == 2:
+#                 time_format = '%H:%M:%S'
+#             elif data.count(':') == 1:
+#                 time_format = '%H:%M'
+#             else:
+#                 time_format = ''
+#
+#             if date_format or time_format:
+#                 if ' ' in data:
+#                     datetime_format = '{} {}'.format(date_format, time_format)
+#                 else:
+#                     datetime_format = '{}{}'.format(date_format, time_format)
+#
+#                 _data = datetime.strptime(data, datetime_format)
+#                 if is_datetime is False:
+#                     _data = int(mktime(_data.timetuple()))
+#             else:
+#                 if is_datetime:
+#                     return None
+#                 return 0
+#     else:
+#         if is_datetime:
+#             return None
+#         return 0
+#
+#     return _data
+#
+#
+# def camelize(uncamelized_str):
+#     if not uncamelized_str:
+#         return uncamelized_str
+#     result = ''.join(i.capitalize() for i in uncamelized_str.split('_'))
+#     result = ''.join((result[0].lower(), result[1:]))
+#     return result
+#
+#
+# def uncamelize(camelized_str):
+#     if not camelized_str:
+#         return camelized_str
+#     lst = []
+#     for index, char in enumerate(camelized_str):
+#         if char.isupper() and index != 0:
+#             lst.append("_")
+#         lst.append(char)
+#
+#     return ''.join(lst).lower()
+#
+#
+# def uncamelize_dict(d):
+#     return {uncamelize(k): v for k, v in d.items()}
+#
+#
+# def camelize_dict(d):
+#     return {camelize(k): v for k, v in d.items()}
+#
+#
+# def get_request_json(is_uncamelize=True):
+#     """
+#     获取 json 传递参数
+#     :return:
+#     """
+#     if request.method.lower() == 'get':
+#         data = request.args.to_dict()
+#     else:
+#         if request.content_encoding and 'gzip' in request.content_encoding:
+#             json_data = zlib.decompress(request.get_data())
+#             data = json.loads(json_data)
+#         else:
+#             data = request.get_json(force=True, silent=True) or {}
+#     if is_uncamelize:
+#         data = {uncamelize(k): v for k, v in data.items()}
+#
+#     g.request_data = strip_value(data)
+#
+#     return g.request_data
+#
+#
+# def get_params():
+#     data = request.args.to_dict()
+#     if request.method.lower() != 'get':
+#         if request.content_encoding and 'gzip' in request.content_encoding:
+#             json_data = zlib.decompress(request.get_data())
+#             data.update(json.loads(json_data))
+#         else:
+#             data.update(request.get_json(force=True, silent=True) or {})
+#     return data

web/commons/meta.py

@@ -0,0 +1,42 @@
+import functools
+
+from nc_http.core import ResponseMeta
+
+
+# 请求已成功，请求所希望的响应头或数据体将随此响应返回。
+OK = ResponseMeta(code=200, http_code=200, description='')
+# 请求已经被实现，而且有一个新的资源已经依据请求的需要而创建。
+CREATED = ResponseMeta(code=201, http_code=201, description='')
+# 资源已经删除。
+NO_CONTENT = ResponseMeta(code=204, http_code=204, description='')
+# 由于包含语法错误，当前请求无法被服务器理解。
+BAD_REQUEST = ResponseMeta(code=400, http_code=400, description='请求数据错误!')
+# 当前请求需要用户验证。
+UNAUTHORIZED = ResponseMeta(code=401, http_code=401, description='授权无效!')
+# 服务器已经理解请求，但是拒绝执行它。
+FORBIDDEN = ResponseMeta(code=403, http_code=403, description='没权限访问,请退出重新登录!')
+# 无权限操作
+OPERATE_FORBIDDEN = ResponseMeta(code=407, http_code=403, description='无操作权限')
+# 请求失败，请求所希望得到的资源未被在服务器上发现。
+NOT_FOUND = ResponseMeta(code=404, http_code=404, description='资源不存在!')
+# 请求行中指定的请求方法不能被用于请求相应的资源。
+METHOD_NOT_ALLOWED = ResponseMeta(code=405, http_code=405, description='方法不存在!')
+# 请求的资源的内容特性无法满足请求头中的条件，因而无法生成响应实体。
+NOT_ACCEPTABLE = ResponseMeta(code=406, http_code=406, description='客户端无效!')
+# 服务器遇到了一个未曾预料的状况，导致了它无法完成对请求的处理。
+INTERNAL_SERVER_ERROR = ResponseMeta(code=500, http_code=500, description='服务器内部错误!')
+# 作为网关或者代理工作的服务器尝试执行请求时，从上游服务器接收到无效的响应。
+BAD_GATEWAY = ResponseMeta(code=502, http_code=502, description='接口错误!')
+# 由于临时的服务器维护或者过载，服务器当前无法处理请求。
+SERVICE_UNAVAILABLE = ResponseMeta(code=503, http_code=503, description='服务暂时不可用!')
+# 未能及时从上游服务器收到响应。
+GATEWAY_TIMEOUT = ResponseMeta(code=504, http_code=504, description='接口超时!')
+
+INVALID_PARAMS = functools.partial(lambda **kwargs: ResponseMeta(
+    code=11001, http_code=400,
+    description='非法请求参数{}'.format(': {}'.format(kwargs['hint_message']) if kwargs.get('hint_message') else ''),
+    **kwargs
+))
+
+FILE_NOT_FOUND = ResponseMeta(code=403, http_code=403, description='未获取到文件')
+

web/commons/models/__init__.py

@@ -0,0 +1,176 @@
+from datetime import datetime
+
+import time
+
+from nc_http.tools.helpers import camelize
+from sqlalchemy.orm.exc import NoResultFound
+
+from commons.models.atomic import Atomic
+
+
+class BaseModel:
+    query = None
+
+    _unique_fields = []
+    _db = None
+
+    __table__ = None
+
+    @classmethod
+    def atomic(cls):
+        return Atomic(db=cls._db)
+
+    @classmethod
+    def get_by_id(cls, id_, session=None):
+        if session:
+            query = session.query(cls)
+        else:
+            query = cls.query
+        result = query.filter((getattr(cls, 'id', None) or cls.__table__.primary_key) == id_).one_or_none()
+        return result
+
+    @classmethod
+    def insert_many(cls, dicts, commit=True, session=None):
+        session = session or cls._db.session()
+        session.bulk_insert_mappings(cls, dicts)
+        if commit:
+            session.commit()
+
+    @classmethod
+    def insert_one(cls, values, commit=True, session=None, return_obj=False):
+        session = session or cls._db.session()
+        if not return_obj:
+            session.bulk_insert_mappings(cls, [values])
+        else:
+            obj = cls()
+            for key, value in values.items():
+                setattr(obj, key, value)
+
+            session.add(obj)
+
+        if commit:
+            session.commit()
+
+        if return_obj:
+            return obj
+
+        return
+
+    @classmethod
+    def update_one(cls, values, commit=True, session=None):
+        session = session or cls._db.session()
+        session.bulk_update_mappings(cls, [values])
+        if commit:
+            session.commit()
+
+    @classmethod
+    def update_many(cls, dicts, commit=True, session=None):
+        session = session or cls._db.session()
+        session.bulk_update_mappings(cls, dicts)
+        if commit:
+            session.commit()
+
+    def to_dict(self, is_camelized=True):
+        _ = {}
+        for column in self.__table__.columns:
+            key = getattr(column, 'quote', None) or column.name
+            if is_camelized:
+                _[camelize(key)] = getattr(self, key, None)
+            else:
+                _[key] = getattr(self, key, None)
+        return _
+
+    @classmethod
+    def select_one(cls, query, fields=None):
+        try:
+            result = query.one()
+        except NoResultFound:
+            return {}
+
+        if not result:
+            return {}
+
+        return dict(zip([i.key for i in fields], result))
+
+    @classmethod
+    def test_one(cls):
+        result = cls.query.first()
+        return result.to_dict()
+
+    @classmethod
+    def now(cls):
+        return int(time.time())
+
+    @classmethod
+    def now_datetime(cls):
+        return datetime.fromtimestamp(int(time.time()))
+
+    @classmethod
+    def get_all(cls):
+        objects = cls.query.all()
+        for obj in objects:
+            yield obj.to_dict()
+
+    def update(self, commit=True, session=None, **kwargs):
+        for key, value in kwargs.items():
+            if hasattr(self, key):
+                setattr(self, key, value)
+        if commit:
+            session = session or self._db.session()
+            session.commit()
+
+        return
+
+    def delete(self, session=None):
+        session = session or self._db.session()
+        session.delete(self)
+
+    @classmethod
+    def get_list(cls, query, paging=None, fields=None):
+        if fields:
+            query = query.with_entities(*fields.values())
+
+        if paging:
+            page = query.paginate(paging['page'], paging['limit'])
+            paging['total'] = page.total
+            result = page.items
+        else:
+            result = query.all()
+
+        if fields:
+            columns = fields.keys()
+            result = [dict(zip(columns, _)) for _ in result]
+        else:
+            result = [_.to_dict() for _ in result]
+
+        if paging:
+            return result, paging
+
+        return result
+
+    @classmethod
+    def set_order_by(cls, query, sorting=None):
+        """
+        配置查询结果排序
+        :param query:
+        :param sorting:
+        :return:
+        """
+        if not sorting or not isinstance(sorting, dict):
+            return query
+
+        order_field = sorting.get('order_field')
+        if not order_field:
+            return query
+
+        order = sorting.get('order')
+        if order == 'desc':
+            query = query.order_by(getattr(cls, order_field).desc())
+        else:
+            query = query.order_by(getattr(cls, order_field))
+
+        return query
+
+
+
+

web/commons/models/asphalt_domestic.py

@@ -0,0 +1,50 @@
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint, func
+
+from commons.models.mixin.calculator import CalculatorMixin
+from core.extensions import db
+
+
+class AsphaltDomestic(db.Model, CalculatorMixin):
+    __tablename__ = 'ASPHALT_DOMESTIC'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    date = Column('DATE', Date, comment='日期')
+    from_ = Column('FROM', Date, comment='数据来源')
+
+    __table_args__ = (
+        UniqueConstraint(name, date, from_,  name='Idx_key'),
+        {'comment': '国产沥青'},
+    )
+
+    @classmethod
+    def get_by_key(cls, name, date, from_):
+        query = cls.query
+        query = query.filter(
+            cls.name == name,
+            cls.date == date,
+            cls.from_ == from_,
+        )
+        return query.one_or_none()
+
+    @classmethod
+    def get_items(cls, date,  name_in):
+        query = cls.query
+        query = query.filter(cls.date == date)
+        query = query.filter(cls.name.in_(name_in))
+        query = query.with_entities(cls.name, func.avg(cls.price))
+        query = query.group_by(cls.name)
+        result = query.all()
+        return result
+
+    def upsert(self):
+        result = self.get_by_key(self.name, self.date, self.from_)
+        if not result:
+            session = db.session
+            session.add(self)
+            session.commit()
+        else:
+            session = db.session
+            self.id = result.id
+            session.add(result)
+            session.commit()

web/commons/models/asphalt_imported.py

@@ -0,0 +1,48 @@
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint, func
+
+from commons.models.mixin.calculator import CalculatorMixin
+from core.extensions import db
+
+
+class AsphaltImported(db.Model, CalculatorMixin):
+    __tablename__ = 'ASPHALT_IMPORTED'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, date, name='Idx_key'),
+        {'comment': '进口沥青'},
+    )
+
+    @classmethod
+    def get_items(cls, year, month, name_in):
+        query = cls.query
+        query = cls.query_by_month(query, year, month)
+        query = query.filter(cls.name.in_(name_in))
+        query = query.with_entities(cls.name, func.avg(cls.price))
+        query = query.group_by(cls.name)
+        result = query.all()
+        return result
+
+    @classmethod
+    def get_by_key(cls, name, date):
+        query = cls.query
+        query = query.filter(
+            cls.name == name,
+            cls.date == date,
+        )
+        return query.one_or_none()
+
+    def upsert(self):
+        result = self.get_by_key(self.name, self.date)
+        if not result:
+            session = db.session
+            session.add(self)
+            session.commit()
+        else:
+            session = db.session
+            self.id = result.id
+            session.add(result)
+            session.commit()

web/commons/models/asphalt_modifier.py

@@ -0,0 +1,36 @@
+import datetime
+
+from dateutil.relativedelta import relativedelta
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint
+
+from core.extensions import db
+
+
+class AsphaltModifier(db.Model):
+    __tablename__ = 'ASPHALT_MODIFIER'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, date, name='Idx_key'),
+        {'comment': '沥青改性剂'},
+    )
+
+    @classmethod
+    def get_query(cls, year=None, month=None, name=None, spec=None, name_in=None):
+        query = cls.query
+        if year and month:
+            start_date = datetime.date(year, month, 1)
+            end_date = start_date + relativedelta(months=1)
+            query = query.filter(cls.date >= start_date)
+            query = query.filter(cls.date < end_date)
+        if name:
+            query = query.filter(cls.name == name)
+        if name_in:
+            query = query.filter(cls.name.in_(name_in))
+        if spec:
+            query = query.filter(cls.spec == spec)
+        return query

web/commons/models/atomic/__init__.py

@@ -0,0 +1,23 @@
+from sqlalchemy.orm import sessionmaker
+
+
+class Atomic:
+    def __init__(self, db, session=None):
+        self.session = session or sessionmaker(bind=db.engine)()  # 此处 session 复用会导致事务异常 应创建 session
+
+    def __enter__(self):
+        return self.session
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        if exc_tb is None:
+            try:
+                self.session.commit()
+            except Exception as e:
+                self.session.rollback()
+                raise e
+            finally:
+                self.session.close()
+        else:
+            return False
+
+        return True

web/commons/models/budget.py

@@ -0,0 +1,20 @@
+from sqlalchemy import Column, Integer, String, Date, Text
+
+from core.extensions import db
+
+
+class Budget(db.Model):
+    __tablename__ = 'BUDGET'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    amount = Column('AMOUNT', Integer, default='', comment='总数')
+    min_amount = Column('MIN_AMOUNT', Integer, default='', comment='最小值')
+    max_amount = Column('MAX_AMOUNT', Integer, default='', comment='最大值')
+    year = Column('YEAR', Integer, default=0, comment='年份')
+    months = Column('MONTHS', Text, default='', comment='月份')
+    date = Column('DATE', Date, comment='日期')
+
+    # __table_args__ = (
+    #     UniqueConstraint(name, date, name='Idx_key'),
+    #     {'comment': '预算'},
+    # )

web/commons/models/budget_item.py

@@ -0,0 +1,20 @@
+from sqlalchemy import Column, Integer, String, Text
+
+from core.extensions import db
+
+
+class BudgetItem(db.Model):
+    __tablename__ = 'BUDGET_ITEM'
+    id = Column('ID', Integer, primary_key=True)
+    budget_id = Column('BUDGET_ID', Integer, default='', comment='预算id')
+    name = Column('NAME', String(128), default='', comment='名称')
+    meta = Column('META', Text, default='', comment='数据')
+    quantity = Column('QUANTITY', Integer, default='', comment='数量')
+    unit = Column('UNIT', Text, default='', comment='单位')
+    unit_Price = Column('UNIT_PRICE', Integer, default='', comment='单价')
+    total_Price = Column('TOTAL_PRICE', Integer, default='', comment='总价')
+
+    # __table_args__ = (
+    #     UniqueConstraint(name, date, name='Idx_key'),
+    #     {'comment': '预算细项'},
+    # )

web/commons/models/cement.py

@@ -0,0 +1,59 @@
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint, func
+
+from commons.models.mixin.calculator import CalculatorMixin
+from core.extensions import db
+
+
+class Cement(db.Model, CalculatorMixin):
+    __tablename__ = 'CEMENT'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    pack = Column('PACK', String(64), default='', comment='包装')
+    source = Column('SOURCE', String(64), default='', comment='产地')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    fluctuating = Column('FLUCTUATING', Numeric(16, 4), default=0, comment='浮动')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, pack, source, date, name='Idx_key'),
+        {'comment': '水泥'},
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def get_items(cls, year, month, spec_in, name_in, pack):
+        query = cls.query
+        query = cls.query_by_month(query, year, month)
+        query = query.filter(cls.pack == pack)
+        query = query.filter(cls.spec.in_(spec_in))
+        query = query.filter(cls.name.in_(name_in))
+        query = query.with_entities(func.avg(cls.price))
+        result = query.all()
+        return result
+
+    @classmethod
+    def get_by_key(cls, name, spec, pack, source, date):
+        query = cls.query
+        query = query.filter(
+            cls.name == name,
+            cls.spec == spec,
+            cls.pack == pack,
+            cls.source == source,
+            cls.date == date,
+        )
+        return query.one_or_none()
+
+    def upsert(self):
+        result = self.get_by_key(self.name, self.spec, self.pack, self.source, self.date)
+        if not result:
+            session = db.session
+            session.add(self)
+            session.commit()
+        else:
+            session = db.session
+            self.id = result.id
+            session.add(result)
+            session.commit()

web/commons/models/data_fujian.py

@@ -0,0 +1,40 @@
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint
+
+from commons.models.mixin.base import BaseModelMixin
+from commons.models.model import Model
+from core.extensions import db
+
+
+class DataFujian(db.Model,  Model, BaseModelMixin):
+    __tablename__ = 'DATA_FUJIAN'
+
+    id = Column('ID', Integer, primary_key=True)
+    number = Column('NUMBER', String(128), default='', comment='编码')
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    unit = Column('UNIT', String(128), default='', comment='单位')
+    price_without_tax = Column('PRICE_WITHOUT_TAX', Numeric(16, 4), default=0, comment='除价格')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='含税价')
+    category = Column('CATEGORY', String(128), default='', comment='分类')
+    year = Column('YEAR', Integer, default=0, comment='年份')
+    month = Column('MONTH', Integer, default=0, comment='月份')
+    city = Column('CITY', String(128), default='', comment='地市')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(year, month, city, name, spec, name='Idx_key'),
+        {'comment': '福建数据'},
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def find_by_key(self):
+        query = DataFujian.query
+        query = query.filter(DataFujian.year == self.year)
+        query = query.filter(DataFujian.month == self.month)
+        query = query.filter(DataFujian.city == self.city)
+        query = query.filter(DataFujian.name == self.name)
+        query = query.filter(DataFujian.spec == self.spec)
+        result = query.one_or_none()
+        return result

web/commons/models/data_guangdong.py

@@ -0,0 +1,30 @@
+from sqlalchemy import Column, Integer, String, Date, UniqueConstraint
+
+from commons.models.mixin.base import BaseModelMixin
+from commons.models.model import Model
+from core.extensions import db
+
+
+class DataGuangdong(db.Model,  Model, BaseModelMixin):
+    __tablename__ = 'DATA_GUANGDONG'
+
+    id = Column('ID', Integer, primary_key=True)
+    url = Column('URL', String(512), default='', comment='下载地址')
+    name = Column('NAME', String(128), default='', comment='名称')
+    source = Column('SOURCE', String(128), default='', comment='来源')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, date, name='Idx_key'),
+        {'comment': '广东数据'},
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def find_by_key(self):
+        query = DataGuangdong.query
+        query = query.filter(DataGuangdong.name == self.name)
+        query = query.filter(DataGuangdong.date == self.date)
+        result = query.one_or_none()
+        return result

web/commons/models/data_zhejiang.py

@@ -0,0 +1,30 @@
+from sqlalchemy import Column, Integer, String, Date, UniqueConstraint
+
+from commons.models.mixin.base import BaseModelMixin
+from commons.models.model import Model
+from core.extensions import db
+
+
+class DataZhejiang(db.Model, Model, BaseModelMixin):
+    __tablename__ = 'DATA_ZHEJIANG'
+
+    id = Column('ID', Integer, primary_key=True)
+    url = Column('URL', String(512), default='', comment='下载地址')
+    name = Column('NAME', String(128), default='', comment='名称')
+    source = Column('SOURCE', String(128), default='', comment='来源')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, date, name='Idx_key'),
+        {'comment': '浙江数据'},
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def find_by_key(self):
+        query = DataZhejiang.query
+        query = query.filter(DataZhejiang.name == self.name)
+        query = query.filter(DataZhejiang.date == self.date)
+        result = query.one_or_none()
+        return result

web/commons/models/fujian_survey.py

@@ -0,0 +1,40 @@
+import datetime
+
+from dateutil.relativedelta import relativedelta
+from sqlalchemy import Column, Integer, String, Date, UniqueConstraint, Numeric
+
+from core.extensions import db
+
+
+class FujianSurvey(db.Model):
+    __tablename__ = 'FUJIAN_SURVEY'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    date = Column('DATE', Date, comment='日期')
+    material_id = Column('MATERIAL_ID', String(128), comment='材料id')
+    unit = Column('UNIT', String(128), comment='单位')
+    brand = Column('BRAND', String(128), comment='品牌')
+    tax = Column('TAX', Integer, comment='税率')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, date, name='Idx_key'),
+        {'comment': '福建省交通工程材料调查表'},
+    )
+
+    @classmethod
+    def get_query(cls, year=None, month=None, name=None, spec=None, name_in=None):
+        query = cls.query
+        if year and month:
+            start_date = datetime.date(year, month, 1)
+            end_date = start_date + relativedelta(months=1)
+            query = query.filter(cls.date >= start_date)
+            query = query.filter(cls.date < end_date)
+        if name:
+            query = query.filter(cls.name == name)
+        if name_in:
+            query = query.filter(cls.name.in_(name_in))
+        if spec:
+            query = query.filter(cls.spec == spec)
+        return query

web/commons/models/fuzhou_highway_bureau.py

@@ -0,0 +1,36 @@
+import datetime
+
+from dateutil.relativedelta import relativedelta
+from sqlalchemy import Column, Integer, String, Date, UniqueConstraint, Numeric
+
+from core.extensions import db
+
+
+class FuzhouHighwayBureau(db.Model):
+    __tablename__ = 'FUZHOU_HIGHWAY_BUREAU'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    date = Column('DATE', Date, comment='日期')
+    material_id = Column('MATERIAL_ID', String(128), comment='材料id')
+    unit = Column('UNIT', String(128), comment='单位')
+    brand = Column('BRAND', String(128), comment='品牌')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, date, name='Idx_key'),
+        {'comment': '福州公路局'},
+    )
+
+    @classmethod
+    def get_query(cls, year, month, name, spec=None):
+        start_date = datetime.date(year, month, 1)
+        end_date = start_date + relativedelta(months=1)
+        query = cls.query
+        query = query.filter(cls.date >= start_date)
+        query = query.filter(cls.date < end_date)
+        if name:
+            query = query.filter(cls.name == name)
+        if spec:
+            query = query.filter(cls.spec == spec)
+        return query

web/commons/models/fuzhou_transportation_bureau.py

@@ -0,0 +1,34 @@
+import datetime
+
+from dateutil.relativedelta import relativedelta
+from sqlalchemy import Column, Integer, String, Date, UniqueConstraint, Numeric
+
+from core.extensions import db
+
+
+class FuzhouTransportationBureau(db.Model):
+    __tablename__ = 'FUZHOU_TRANSPORTATION_BUREAU'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    date = Column('DATE', Date, comment='日期')
+    material_id = Column('MATERIAL_ID', String(128), comment='材料id')
+    unit = Column('UNIT', String(128), comment='单位')
+    brand = Column('BRAND', String(128), comment='品牌')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, date, name='Idx_key'),
+        {'comment': '福州交通局'},
+    )
+
+    @classmethod
+    def get_query(cls, year, month, name):
+        start_date = datetime.date(year, month, 1)
+        end_date = start_date + relativedelta(months=1)
+        query = cls.query
+        query = query.filter(cls.date >= start_date)
+        query = query.filter(cls.date < end_date)
+        if name:
+            query = query.filter(cls.name == name)
+        return query

web/commons/models/local_material.py

@@ -0,0 +1,24 @@
+from sqlalchemy import Column, Integer, String, Date, UniqueConstraint, Numeric, Text
+
+from core.extensions import db
+
+
+class LocalMaterial(db.Model):
+    __tablename__ = 'LOCAL_MATERIAL'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    city = Column('CITY', String(128), default='', comment='地市')
+    county = Column('COUNTY', String(128), default='', comment='区县')
+    material_id = Column('MATERIAL_ID', String(128), comment='材料id')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    unit = Column('UNIT', String(128), default='', comment='单位')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    price_without_tax = Column('PRICE_WITHOUT_TAX', Numeric(16, 4), default=0, comment='除税价')
+    date = Column('DATE', Date, comment='日期')
+    position = Column('POSITION', String(256), comment='位置')
+    remark = Column('REMARK', Text, comment='备注')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, city, county, date, name='Idx_key'),
+        {'comment': '地材'},
+    )

web/commons/models/material.py

@@ -0,0 +1,44 @@
+from sqlalchemy import Column, Integer, String
+
+from commons.models.mixin.operation_track import OperationTrackMixin
+from core.extensions import db
+
+
+class Material(db.Model, OperationTrackMixin):
+    id = Column('ID', String(128), primary_key=True)
+    parent_id = Column('PARENT_ID', String(128))
+    category_1 = Column('CATEGORY1', String(128), default='', comment='分类1')
+    category_2 = Column('CATEGORY2', String(128), default='', comment='分类2')
+    category_3 = Column('CATEGORY3', String(128), default='', comment='分类3')
+    category_4 = Column('CATEGORY4', String(128), default='', comment='分类4')
+    name = Column('NAME', String(128), default='', comment='名称')
+    unit = Column('UNIT', String(128), default='', comment='单位')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    tax = Column('TAX', Integer, default=0, comment='税率（%）')
+    is_builtin = Column('IS_BUILTIN', Integer, default=0, comment='是否初始内建类型（不允许删除）')
+    type = Column('TYPE', Integer, default=0, comment='材料类别（主材、地材）')
+    is_tree = Column('IS_TREE', Integer, default=0, comment='是否树')
+
+    __tablename__ = 'MATERIAL'
+    __table_args__ = (
+        {'comment': '材料'},
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def get_by_key(cls, id):
+        return cls.query.filter(cls.id == id).one_or_none()
+
+    def upsert(self):
+        result = self.get_by_key(self.id)
+        if not result:
+            session = db.session
+            session.add(self)
+            session.commit()
+        else:
+            session = db.session
+            self.id = result.id
+            session.add(result)
+            session.commit()

web/commons/models/material_task.py

@@ -0,0 +1,22 @@
+from sqlalchemy import Column, Integer, String
+from sqlalchemy.dialects.mysql import MEDIUMTEXT
+
+from commons.models.mixin.operation_track import OperationTrackMixin
+from commons.models.model import Model
+from core.extensions import db
+
+
+class MaterialTask(db.Model, OperationTrackMixin, Model):
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='任务名称')
+    status = Column('STATUS', Integer, default=0, comment='状态（待采集、已采集、采集中）')
+    file = Column('FILE', String(256), default='', comment='文件路径')
+    type = Column('TYPE', Integer, default=0, comment='类型（网络爬取、文件上传）')
+    year = Column('YEAR', Integer, default=0, comment='采集年份')
+    month = Column('MONTH', Integer, default=0, comment='采集月份')
+    content = Column('CONTENT', MEDIUMTEXT, comment='数据内容')
+
+    __tablename__ = 'MATERIAL_TASK'
+    __table_args__ = (
+        {'comment': '采集任务'},
+    )

web/commons/models/mixin/base.py

@@ -0,0 +1,13 @@
+class BaseModelMixin:
+
+    def find_by_key(self):
+        ...
+
+    def upsert(self):
+        result = self.find_by_key()
+        session = self._db.session
+        if result:
+            session.delete(result)
+            session.flush()
+        session.add(self)
+        session.commit()

web/commons/models/mixin/calculator.py

@@ -0,0 +1,28 @@
+class CalculatorMixin:
+    date = None
+
+    @classmethod
+    def query_by_month(cls, query, year, month):
+        last_month_year, last_month = Helper.get_last_month(year, month)
+        query = query.filter(cls.date >= datetime.date(last_month_year, last_month, 26))
+        query = query.filter(cls.date <= datetime.date(year, month, 25))
+        return query
+
+    def upsert(self, *args, **kwargs):
+        result = self.get_by_key(*args, **kwargs)
+        if not result:
+            session = db.session
+            session.add(self)
+            session.commit()
+        else:
+            session = db.session
+            self.id = result.id
+            session.add(result)
+            session.commit()
+
+    @classmethod
+    def get_last_date_from(cls, date):
+        query = cls.query.filter(cls.date <= date)
+        query = query.with_entities(func.max(cls.date))
+        result = query.one_or_none()
+        return result[0]

web/commons/models/mixin/es_source.py

@@ -0,0 +1,23 @@
+class EsSourceMixin:
+    __table__ = None
+    __tablename__ = ''
+
+    def to_es(self):
+        primary_key_name = [str(i.name) for i in self.__table__.columns if i.primary_key][0]
+        es_data = {
+            'meta': {
+                'id': '{}/{}'.format(self.__tablename__, getattr(self, primary_key_name)),
+            },
+            'table': self.__tablename__,
+            'create_time': self.now() * 1000,
+        }
+        return es_data
+
+    @classmethod
+    def get_es_data(cls):
+        for item in cls.query.yield_per(500):
+            yield item.to_es()
+
+    @classmethod
+    def es_join_str(cls, str_list):
+        return ' '.join(str(i or '') for i in str_list if i)

web/commons/models/mixin/insensitive.py

@@ -0,0 +1,51 @@
+from nc_http.tools.helpers import camelize
+
+
+class InsensitiveMixin:
+    __table__ = None
+    _sensitive_columns = []
+
+    def to_insensitive_dict(self, is_camelized=True):
+        """
+        对象转字典且不带敏感数据 <例：密码、密钥、手机号...>
+        :return:
+        """
+        if self._sensitive_columns:
+            _ = {}
+            for column in self.__table__.columns:
+                if column in self._sensitive_columns:
+                    continue
+                key = getattr(column, 'quote', None) or column.name
+                if is_camelized:
+                    _[camelize(key)] = getattr(self, key, None)
+                else:
+                    _[key] = getattr(self, key, None)
+        else:
+            _ = self.to_dict()
+        return _
+
+    @classmethod
+    def get_list(cls, query, paging=None, fields=None):
+        if fields:
+            query = query.with_entities(*fields.values())
+
+        if paging:
+            page = query.paginate(paging['page'], paging['limit'])
+            paging['total'] = page.total
+            result = page.items
+        else:
+            result = query.all()
+
+        if fields:
+            columns = fields.keys()
+            result = [dict(zip(columns, _)) for _ in result]
+        else:
+            if cls._sensitive_columns:  # 数据脱敏
+                result = [_.to_insensitive_dict() for _ in result]
+            else:
+                result = [_.to_dict() for _ in result]
+
+        if paging:
+            return result, paging
+
+        return result

web/commons/models/mixin/operation_track.py

@@ -0,0 +1,30 @@
+from datetime import datetime
+
+from sqlalchemy import Column, String, DateTime
+
+
+class OperationTrackMixin:
+    update_user_id = Column('update_user_id', String(64), comment='最后更新人id')
+    update_user_name = Column('update_user_name', String(64), comment='最后更新人名称')
+    update_time = Column('update_time', DateTime, default=datetime.now, comment='最后更新时间')
+    create_user_id = Column('create_user_id', String(64), comment='创建人id')
+    create_user_name = Column('create_user_name', String(64), comment='创建人名称')
+    create_time = Column('create_time', DateTime, default=datetime.now, comment='创建时间')
+    delete_user_id = Column('delete_user_id', String(64), comment='删除人id')
+    delete_user_name = Column('delete_user_name', String(64), comment='删除人名称')
+    delete_time = Column('delete_time', DateTime, comment='删除时间')
+
+    def track_delete(self, user_id, user_name=None):
+        self.delete_user_id = user_id
+        self.delete_user_name = user_name
+        self.delete_time = datetime.now()
+
+    def track_create(self, user_id, user_name=None):
+        self.create_user_id = user_id
+        self.create_user_name = user_name
+        self.create_time = datetime.now()
+
+    def track_update(self, user_id, user_name=None):
+        self.update_user_id = user_id
+        self.update_user_name = user_name
+        self.update_time = datetime.now()

web/commons/models/mixin/steel.py

@@ -0,0 +1,25 @@
+class SteelMixin(CalculatorMixin):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def get_by_key(cls, name, spec, material, source, date):
+        spec = spec or ''
+        material = material or ''
+        source = source or ''
+        query = cls.query
+        query = query.filter(cls.name == name).filter(cls.spec == spec).filter(cls.material == material)
+        query = query.filter(cls.source == source).filter(cls.date == date)
+        return query.one_or_none()
+
+    def upsert(self):
+        result = self.get_by_key(self.name, self.spec, self.material, self.source, self.date)
+        if not result:
+            session = db.session
+            session.add(self)
+            session.commit()
+        else:
+            session = db.session
+            self.id = result.id
+            session.add(result)
+            session.commit()

web/commons/models/model.py

@@ -0,0 +1,6 @@
+from commons.models import BaseModel
+from core.extensions import db
+
+
+class Model(BaseModel):
+    _db = db

web/commons/models/oil.py

@@ -0,0 +1,48 @@
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint
+
+from commons.models.mixin.calculator import CalculatorMixin
+from commons.models.model import Model
+from core.extensions import db
+
+
+class Oil(db.Model,  Model, CalculatorMixin):
+    __tablename__ = 'OIL'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, date, name='Idx_key'),
+        {'comment': '成品油'},
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def get_items(cls, year, month, name_in):
+        query = cls.query
+        query = cls.query_by_month(query, year, month)
+        query = query.filter(cls.name.in_(name_in))
+        query = query.order_by(cls.date)
+        result = query.all()
+        return result
+
+    @classmethod
+    def get_by_key(cls, name, date):
+        query = cls.query
+        query = query.filter(cls.name == name).filter(cls.date == date)
+        return query.one_or_none()
+
+    def upsert(self):
+        result = self.get_by_key(self.name, self.date)
+        if not result:
+            session = db.session
+            session.add(self)
+            session.commit()
+        else:
+            session = db.session
+            self.id = result.id
+            session.add(result)
+            session.commit()

web/commons/models/price_publish.py

@@ -0,0 +1,81 @@
+import datetime
+
+from dateutil.relativedelta import relativedelta
+from sqlalchemy import Column, Integer, String, Numeric
+
+from commons.models.mixin.base import BaseModelMixin
+from commons.models.mixin.operation_track import OperationTrackMixin
+from core.extensions import db
+
+
+class PricePublish(db.Model, OperationTrackMixin, BaseModelMixin):
+    id = Column('ID', Integer, primary_key=True)
+    year = Column('YEAR', Integer, default='', comment='统计年份')
+    month = Column('MONTH', Integer, default='', comment='统计月份')
+    material_id = Column('MATERIAL_ID', String(128), default='', comment='编号')
+    name = Column('NAME', String(128), default='', comment='材料名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    price_fuzhou = Column('PRICE_FUZHOU', Numeric(16, 4), default=0, comment='福州价格')
+    price_xiamen = Column('PRICE_XIAMEN', Numeric(16, 4), default=0, comment='厦门价格')
+    price_putian = Column('PRICE_PUTIAN', Numeric(16, 4), default=0, comment='莆田价格')
+    price_sanming = Column('PRICE_SANMING', Numeric(16, 4), default=0, comment='三明价格')
+    price_quanzhou = Column('PRICE_QUANZHOU', Numeric(16, 4), default=0, comment='泉州价格')
+    price_zhangzhou = Column('PRICE_ZHANGZHOU', Numeric(16, 4), default=0, comment='漳州价格')
+    price_nanpin = Column('PRICE_NANPIN', Numeric(16, 4), default=0, comment='南平价格')
+    price_longyan = Column('PRICE_LONGYAN', Numeric(16, 4), default=0, comment='龙岩价格')
+    price_ningde = Column('PRICE_NINGDE', Numeric(16, 4), default=0, comment='宁德价格')
+    price_pintan = Column('PRICE_PINTAN', Numeric(16, 4), default=0, comment='平潭价格')
+    tax = Column('TAX', Numeric(4, 2), default=0, comment='税率')
+    status = Column('STATUS', Integer, default=0, comment='状态')
+    type = Column('TYPE', Integer, default=0, comment='类型')
+    unit = Column('UNIT', String(128), default='', comment='单位')
+
+    __tablename__ = 'PRICE_PUBLISH'
+    __table_args__ = (
+        {'comment': '发布价格'},
+    )
+
+    @classmethod
+    def get_by_key(cls, name, year, month, type):
+        query = cls.query
+        query = query.filter(cls.name == name).filter(cls.year == year).filter(cls.month == month).filter(
+            cls.type == type)
+        return query.one_or_none()
+
+    def find_by_key(self):
+        cls = self.__class__
+        query = cls.query
+        query = query.filter(cls.year == self.year)
+        query = query.filter(cls.month == self.month)
+        query = query.filter(cls.name == self.name)
+        query = query.filter(cls.type == self.type)
+        result = query.one_or_none()
+        return result
+
+    # @classmethod
+    # def query_previous_month(cls, query, start_date: datetime.date, count=6):
+    #     end_date = start_date + relativedelta(months=1)
+    #     query = query.filter(cls.date >= end_date - relativedelta(months=count))
+    #     query = query.filter(cls.date < end_date)
+    #     return query
+
+    @classmethod
+    def query_previous_month(cls, query, start_date: datetime.date, count=6):
+        condition = False
+        for i in range(count):
+            date = start_date - relativedelta(months=i)
+            condition = condition or (cls.year == date.year and cls.month == date.month)
+        query.filter(condition)
+        return query
+
+    @classmethod
+    def get_query(cls, year=None, month=None, name_in=None):
+        query = cls.query
+        if year:
+            query = query.filter(cls.year == year)
+        if month:
+            query = query.filter(cls.month == month)
+        if name_in:
+            query = query.filter(cls.name.in_(name_in))
+        return query

web/commons/models/price_result.py

@@ -0,0 +1,60 @@
+from sqlalchemy import Column, Integer, String, Numeric
+
+from commons.models.mixin.base import BaseModelMixin
+from commons.models.mixin.operation_track import OperationTrackMixin
+from core.extensions import db
+
+
+class PriceResult(db.Model, OperationTrackMixin, BaseModelMixin):
+    id = Column('ID', Integer, primary_key=True)
+    material_id = Column('MATERIAL_ID', String(128), default='', comment='编号')
+    name = Column('NAME', String(128), default='', comment='材料名称')
+    year = Column('YEAR', Integer, default='', comment='统计年份')
+    month = Column('MONTH', Integer, default='', comment='统计月份')
+    price_ftb = Column('PRICE_FTB', Numeric(16, 4), default=0, comment='福州交通局价格')
+    fluctuating_ftb = Column('FLUCTUATING_FTB', Numeric(16, 4), default=0, comment='福州交通局浮动')
+    price_ss = Column('PRICE_SS', Numeric(16, 4), default=0, comment='三明钢铁价格')
+    fluctuating_ss = Column('FLUCTUATING_SS', Numeric(16, 4), default=0, comment='三明钢铁浮动')
+    price_fhb = Column('PRICE_FHB', Numeric(16, 4), default=0, comment='福州公路局价格')
+    fluctuating_fhb = Column('FLUCTUATING_FHB', Numeric(16, 4), default=0, comment='福州公路局浮动')
+    price_network = Column('PRICE_NETWORK', Numeric(16, 4), default=0, comment='网络价格')
+    fluctuating_network = Column('FLUCTUATING_NETWORK', Numeric(16, 4), default=0, comment='网络浮动')
+    price_survey = Column('PRICE_SURVEY', Numeric(16, 4), default=0, comment='调查价格')
+    fluctuating_survey = Column('FLUCTUATING_SURVEY', Numeric(16, 4), default=0, comment='调查浮动')
+    price_last_month = Column('PRICE_LAST_MONTH', Numeric(16, 4), default=0, comment='上月发布价格')
+    price_calculate = Column('PRICE_CALCULATE', Numeric(16, 4), default=0, comment='计算价格')
+    price_recommend = Column('PRICE_RECOMMEND', Numeric(16, 4), default=0, comment='推荐价格')
+    fluctuating_recommend = Column('FLUCTUATING_RECOMMEND', Numeric(16, 4), default=0, comment='推荐浮动')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    unit = Column('UNIT', String(128), default='', comment='单位')
+
+    __tablename__ = 'PRICE_RESULT'
+    __table_args__ = (
+        {'comment': '计算结果'},
+    )
+
+    @classmethod
+    def get_by_key(cls, name, year, month):
+        query = cls.query
+        query = query.filter(cls.name == name).filter(cls.year == year).filter(cls.month == month)
+        return query.one_or_none()
+
+    def find_by_key(self):
+        cls = self.__class__
+        query = cls.query
+        query = query.filter(cls.year == self.year)
+        query = query.filter(cls.month == self.month)
+        query = query.filter(cls.name == self.name)
+        result = query.one_or_none()
+        return result
+
+    @classmethod
+    def get_query(cls, year, month, name_in=None, name=None):
+        query = cls.query
+        query = query.filter(cls.year == year)
+        query = query.filter(cls.month == month)
+        if name_in:
+            query = query.filter(cls.name.in_(name_in))
+        if name:
+            query = query.filter(cls.name == name)
+        return query

web/commons/models/sanming_steel.py

@@ -0,0 +1,35 @@
+import datetime
+
+from dateutil.relativedelta import relativedelta
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint
+
+from core.extensions import db
+
+
+class SanmingSteel(db.Model):
+    __tablename__ = 'SANMING_STEEL'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    material = Column('MATERIAL', String(64), default='', comment='材质')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    fluctuating = Column('FLUCTUATING', Numeric(16, 4), default=0, comment='浮动')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, material, date, name='Idx_key'),
+        {'comment': '三明钢铁'},
+    )
+
+    @classmethod
+    def get_query(cls, year, month, name, spec):
+        start_date = datetime.date(year, month, 1)
+        end_date = start_date + relativedelta(months=1)
+        query = cls.query
+        query = query.filter(cls.date >= start_date)
+        query = query.filter(cls.date < end_date)
+        if name:
+            query = query.filter(cls.name == name)
+        if spec:
+            query = query.filter(cls.spec == spec)
+        return query

web/commons/models/steel_plate.py

@@ -0,0 +1,33 @@
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint, func
+
+from commons.models.mixin.steel import SteelMixin
+from core.extensions import db
+
+
+class SteelPlate(db.Model, SteelMixin):
+    __tablename__ = 'STEEL_PLATE'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    material = Column('MATERIAL', String(64), default='', comment='材质')
+    source = Column('SOURCE', String(64), default='', comment='产地')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    fluctuating = Column('FLUCTUATING', Numeric(16, 4), default=0, comment='浮动')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, material, source, date,  name='Idx_key'),
+        {'comment': '中厚板'},
+    )
+
+    @classmethod
+    def get_items(cls, year, month, spec_in=('12', '16-20', '22-28'), source_in=('三钢闽光',), name_in=('普中板', '工字钢')):
+        query = cls.query
+        query = cls.query_by_month(query, year, month)
+        query = query.filter(cls.spec.in_(spec_in))
+        query = query.filter(cls.source.in_(source_in))
+        query = query.filter(cls.name.in_(name_in))
+        query = query.with_entities(cls.spec, func.avg(cls.price))
+        query = query.group_by(cls.spec)
+        result = query.all()
+        return result

web/commons/models/steel_rebar.py

@@ -0,0 +1,44 @@
+import datetime
+
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint, func
+
+from calculators import Helper
+from commons.models.mixin.steel import SteelMixin
+from core.extensions import db
+
+
+class SteelRebar(db.Model, SteelMixin):
+    __tablename__ = 'STEEL_REBAR'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    material = Column('MATERIAL', String(64), default='', comment='材质')
+    source = Column('SOURCE', String(64), default='', comment='产地')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    fluctuating = Column('FLUCTUATING', Numeric(16, 4), default=0, comment='浮动')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, material, source, date,  name='Idx_key'),
+        {'comment': '钢筋'},
+    )
+
+    @classmethod
+    def query_by_month(cls, query, year, month):
+        last_month_year, last_month = Helper.get_last_month(year, month)
+        query = query.filter(cls.date >= datetime.date(last_month_year, last_month, 26))
+        query = query.filter(cls.date <= datetime.date(year, month, 25))
+        return query
+
+    @classmethod
+    def get_items(cls, year, month, material, source='三钢闽光', name_in=('高线', '螺纹钢')):
+        query = cls.query
+        query = cls.query_by_month(query, year, month)
+        query = query.filter(cls.material == material)
+        query = query.filter(cls.source == source)
+        query = query.filter(cls.name.in_(name_in))
+        query = query.with_entities(cls.spec, cls.material, func.avg(cls.price))
+        query = query.group_by(cls.spec, cls.material)
+        result = query.all()
+
+        return result

web/commons/models/steel_section.py

@@ -0,0 +1,32 @@
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint, func
+
+from commons.models.mixin.steel import SteelMixin
+from core.extensions import db
+
+
+class SteelSection(db.Model, SteelMixin):
+    __tablename__ = 'STEEL_SECTION'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    material = Column('MATERIAL', String(64), default='', comment='材质')
+    source = Column('SOURCE', String(64), default='', comment='产地')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    fluctuating = Column('FLUCTUATING', Numeric(16, 4), default=0, comment='浮动')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, material, source, date,  name='Idx_key'),
+        {'comment': '型钢'},
+    )
+
+    @classmethod
+    def get_items(cls, year, month, spec_in=('140*140*12', '25#'), source_in=('唐山弘泰', '唐山正丰'), name_in=('角钢', '工字钢')):
+        query = cls.query
+        query = cls.query_by_month(query, year, month)
+        query = query.filter(cls.spec.in_(spec_in))
+        query = query.filter(cls.source.in_(source_in))
+        query = query.filter(cls.name.in_(name_in))
+        query = query.with_entities(func.avg(cls.price))
+        result = query.all()
+        return result

web/commons/models/steel_strand.py

@@ -0,0 +1,32 @@
+from sqlalchemy import Column, Integer, String, Numeric, Date, UniqueConstraint, func
+
+from commons.models.mixin.steel import SteelMixin
+from core.extensions import db
+
+
+class SteelStrand(db.Model, SteelMixin):
+    __tablename__ = 'STEEL_STRAND'
+    id = Column('ID', Integer, primary_key=True)
+    name = Column('NAME', String(128), default='', comment='名称')
+    spec = Column('SPEC', String(128), default='', comment='规格')
+    material = Column('MATERIAL', String(64), default='', comment='材质')
+    source = Column('SOURCE', String(64), default='', comment='产地')
+    price = Column('PRICE', Numeric(16, 4), default=0, comment='价格')
+    fluctuating = Column('FLUCTUATING', Numeric(16, 4), default=0, comment='浮动')
+    date = Column('DATE', Date, comment='日期')
+
+    __table_args__ = (
+        UniqueConstraint(name, spec, material, source, date, name='Idx_key'),
+        {'comment': '钢绞线'},
+    )
+
+    @classmethod
+    def get_items(cls, year, month, material_in=('SWRH82B',), name_in=('弹簧钢',)):
+        query = cls.query
+        query = cls.query_by_month(query, year, month)
+        query = query.filter(cls.material.in_(material_in))
+        query = query.filter(cls.name.in_(name_in))
+        query = query.with_entities(cls.material, func.avg(cls.price))
+        query = query.group_by(cls.material)
+        result = query.all()
+        return result

web/commons/services/data.py

@@ -0,0 +1,50 @@
+import json
+from multiprocessing.pool import Pool
+
+from commons.constants.material_task import MaterialTaskType
+from commons.models.data_fujian import DataFujian
+from commons.models.data_guangdong import DataGuangdong
+from commons.models.data_zhejiang import DataZhejiang
+from commons.models.oil import Oil
+from spiders import run_spider
+from spiders.data_fujian import DataFujianSpider
+from spiders.data_guangdong import DataGuangdongSpider
+from spiders.date_zhejiang import DataZhejiangSpider
+from spiders.oil import OilSpider
+
+
+class DataService:
+    spiders = {
+        MaterialTaskType.OTHER_ZHEJIANG: DataZhejiangSpider,
+        MaterialTaskType.OTHER_GUANGZHOU: DataGuangdongSpider,
+        MaterialTaskType.OTHER_YUNNAN: None,
+        MaterialTaskType.FUJIAN_DEPARTMENT: DataFujianSpider,
+        MaterialTaskType.OIL: OilSpider,
+    }
+    models = {
+        MaterialTaskType.OTHER_ZHEJIANG: DataZhejiang,
+        MaterialTaskType.OTHER_GUANGZHOU: DataGuangdong,
+        MaterialTaskType.OTHER_YUNNAN: None,
+        MaterialTaskType.FUJIAN_DEPARTMENT: DataFujian,
+        MaterialTaskType.OIL: Oil,
+    }
+
+    @classmethod
+    def get_content(cls, type):
+        spider_class = cls.spiders.get(type)
+        if not spider_class:
+            return []
+
+        pool = Pool(processes=1)
+        result = pool.apply_async(run_spider, (spider_class,))
+        pool.close()
+        pool.join()
+        file_path = result.get()
+
+        # file_path = run_spider(spider_class)
+        content = json.loads(open(file_path, 'r', encoding='utf-8').read())
+        model_class = cls.models.get(type)
+        for item in content:
+            model_class(**item).upsert()
+        return content
+

web/commons/vendors/__init__.py

@@ -0,0 +1,66 @@
+from urllib.parse import urlencode
+
+import requests
+
+
+class ApiException(Exception):
+    def __init__(self, code, message):
+        self.code = code
+        self.message = message
+        super().__init__()
+
+    def __str__(self):
+        return self.message
+
+
+class BaseClient:
+    gateway = None
+    timeout = 10
+    max_retries = 3
+
+    def _build_url(self, interface, params, query=None, files=None):
+        method, uri = interface
+
+        query_params = query or {}
+        if params and (method == 'get' or files):
+            query_params.update(params)
+
+        api_url = '{}/{}?{}'.format(self.gateway, uri, urlencode(query_params))
+        return api_url
+
+    def _get_headers(self):
+        return None
+
+    def call(self, interface, params=None, query=None, files=None, max_retries=max_retries, without_unpack=False):
+        method, *_ = interface
+        api_url = self._build_url(interface, params, query, files)
+        headers = self._get_headers()
+        try:
+            if method == 'get':
+                response = requests.request(method, api_url, headers=headers, timeout=self.timeout)
+            else:
+                response = requests.request(
+                    method, api_url, headers=headers, timeout=self.timeout, json=params, files=files
+                )
+        except Exception as e:
+            if max_retries > 0:
+                return self.call(interface, params, files=files, max_retries=max_retries - 1)
+            raise ApiException(504, "GATEWAY TIMEOUT") from e
+
+        if without_unpack:
+            return response
+
+        try:
+            data = self.unpack(response)
+        except ApiException as e:
+            raise e
+        except Exception as e:
+            print('BaseClient.call | interface | {}'.format(interface))
+            print('BaseClient.call | api_url | {}'.format(api_url))
+            raise ApiException(502, "BAD GATEWAY") from e
+
+        return data
+
+    @staticmethod
+    def unpack(response):
+        pass

web/config.py

@@ -0,0 +1,79 @@
+import logging
+import os
+
+PORT = 7778
+
+# 文件路径
+APP_ROOT = os.path.abspath(os.path.dirname(__file__))
+CACHE_DIR = os.path.join(APP_ROOT, 'cache')
+FILE_DIR = os.path.join(APP_ROOT, 'files')
+TEMP_DIR = os.getenv('TEMP', '/tmp')
+API_ROOT = ''
+
+# 调试模式
+APP_MODE = os.getenv('APP_MODE', 'Development')  # Development
+DEBUG = bool(APP_MODE in ['Development', 'Testing'])
+
+# 密码安全
+SECRET_KEY = 'nN<]9Ss32b%GCc,T8q*.^+65A47@=)'
+TOKEN_SALT = '20140606'
+
+# 日志
+LOG_DIR = '/tmp'
+LOG_FORMAT = '%(thread)d %(asctime)s %(filename)s:%(lineno)d %(levelname)s: %(message)s'
+stream_handler = logging.StreamHandler()
+stream_handler.setFormatter(logging.Formatter(LOG_FORMAT))
+logging.getLogger().addHandler(stream_handler)
+logging.getLogger().setLevel(logging.DEBUG)  # 默认错误日志级别
+
+# 当前服务URL地址
+HTTP_BASE_URL = os.getenv('HTTP_BASE_URL', 'http://192.168.1.3:7072')
+HTTPS_BASE_URL = os.getenv('HTTPS_BASE_URL', '')
+
+# 数据库
+os.environ['NLS_LANG'] = 'SIMPLIFIED CHINESE_CHINA.UTF8'  # oracle client 编码
+SQLALCHEMY_DATABASE_URI = 'mysql+mysqlconnector://root:Xxs123456@192.168.1.3:4306/material_manage?charset=utf8mb4&auth_plugin=mysql_native_password'
+SQLALCHEMY_TRACK_MODIFICATIONS = False  # 设置是否跟踪数据库的修改情况，一般不跟踪
+SQLALCHEMY_ECHO = True  # 数据库操作时是否显示原始SQL语句，一般打开，后台要日志
+SQLALCHEMY_BINDS = {
+    # 'arcgis': 'postgres://postgres:lyh123456!@192.168.137.1:7432/dzonemap',
+}
+SQLALCHEMY_POOL_RECYCLE = 10 * 60  # 空闲该时间后回收数据库连接
+
+# ElasticSearch
+ES_HOSTS = os.getenv('ES_HOSTS', 'http://192.168.137.1:9200')
+
+# Redis
+REDIS_HOST = os.getenv('REDIS_HOST', '192.168.137.1')
+REDIS_PORT = int(os.getenv('REDIS_PORT', '6379'))
+REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', '')
+REDIS_DB = int(os.getenv('REDIS_DATABASE', '1'))
+
+# Celery
+CELERY_BROKER_URL = 'redis://{}{}:{}/{}'.format(
+    ':{}@'.format(REDIS_PASSWORD) if REDIS_PASSWORD else '',
+    REDIS_HOST,
+    REDIS_PORT,
+    REDIS_DB
+)
+BROKER_URL = CELERY_BROKER_URL
+CELERY_RESULT_BACKEND = CELERY_BROKER_URL
+
+# 服务
+SERVER_HOST = {
+    'arcpy': 'http://192.168.137.1:8777',
+    'wmts': 'http://192.168.137.1:7072',
+    'geoserver': 'https://xxslyh.cn',
+    'tianditu': 'https://t{no}.tianditu.gov.cn',
+    'tianditu_fj': 'http://192.168.137.1:7072',
+    'tianditu_fj_s0': 'http://s0.fjmap.net',
+    'file': 'http://192.168.1.3:7072',
+    'geological_map_service': 'http://192.168.137.1:7026',
+    'arcgis_server': 'http://192.168.137.247:6080',
+    'map_service': 'http://192.168.137.1:7028',
+}
+
+try:
+    from local_config import *
+except ImportError:
+    pass

web/core/app.py

@@ -0,0 +1,12 @@
+""" 程序入口"""
+from nc_http.tools.reverse_proxied import ReverseProxied
+
+from core import conf
+from core.factory import ApiApp
+
+app = ApiApp()
+
+app.wsgi_app = ReverseProxied(app.wsgi_app)
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=conf.PORT, debug=conf.DEBUG)

web/core/conf.py

@@ -0,0 +1,13 @@
+# try:
+#     from web.config import *
+# except ModuleNotFoundError as e:
+#     pass
+#
+# APP_PATH = os.path.abspath(os.path.dirname(__file__))
+#
+# SECRET_KEY = "tE7?aS.1-7sP4dfvlE"
+# TOKEN_SALT = "hi,_tool"
+# QR_CODE_SALT = "code_qr"
+#
+# # 监听端口
+# PORT = 7778

web/core/data_zhejiang.json

@@ -0,0 +1,34 @@
+[
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=f9850959bc874109bbf969d4f1407765.pdf", "name": "《质监与造价》价格信息专辑2022年第二期（总第217期）.pdf", "source": "省交通工程管理中心", "date": "2022-03-10"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=a5588454a5d2493e84b1da41e355d8c4.pdf", "name": "《质监与造价》价格信息专辑2022年第六期（总第221期）.pdf", "source": "省交通工程管理中心", "date": "2022-07-11"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=89f18f9f1a104e86b1ebece9d4f54e9a.pdf", "name": "《质监与造价》价格信息专辑2022年第四期（总第219期）.pdf", "source": "省交通工程管理中心", "date": "2022-05-12"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=fb4970a1c47843acaeedcdc28d234542.pdf", "name": "《质监与造价》价格信息专辑2022年第三期（总第218期）.pdf", "source": "省交通工程管理中心", "date": "2022-04-11"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=6813301dc7664bce89531ec9e3b11f26.pdf", "name": "《质监与造价》价格信息专辑2022年第九期（总第224期）.pdf", "source": "省交通工程管理中心", "date": "2022-10-10"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=115577c096fc4b9dbd5226d9c631ea0c.pdf", "name": "《质监与造价》价格信息专辑2022年第五期（总第220期）.pdf", "source": "省交通工程管理中心", "date": "2022-06-10"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=492109fa11004378b8d8ddd876857a3b.pdf", "name": "《质监与造价》价格信息专辑2022年第十期（总第225期）.pdf", "source": "省交通工程管理中心", "date": "2022-11-14"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=7b65f691b00446bea1364fef476a74e1.pdf", "name": "《质监与造价》价格信息专辑2022年第七期（总第222期）.pdf", "source": "省交通工程管理中心", "date": "2022-08-12"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=b6819feaa98f4dc48890b6f50bf5edb8.pdf", "name": "《质监与造价》价格信息专辑2022年第八期（总第223期）.pdf", "source": "省交通工程管理中心", "date": "2022-09-14"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=3de209f960fd4115b5b27666dfd8a076.pdf", "name": "《质监与造价》价格信息专辑2022年第十一期（总第226期）.pdf", "source": "省交通工程管理中心", "date": "2022-12-19"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=04591e88c9894076a5afe096fa6bb8aa.pdf", "name": "《质监与造价》价格信息专辑2022年第十二期（总第227期）.pdf", "source": "省交通工程管理中心", "date": "2023-01-19"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=61f09bc9376946bba47824301d082a0f.pdf", "name": "《质监与造价》价格信息专辑2023年第三期（总第230期）.pdf", "source": "省交通工程管理中心", "date": "2023-04-18"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=93ac1fa626f04a53990ab5fd5cf12435.pdf", "name": "《质监与造价》价格信息专辑2021年第三期（总第206期）.pdf", "source": "省交通工程管理中心", "date": "2021-04-15"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=ea07c46c718041af8a6138e15dec98e5.pdf", "name": "《质监与造价》价格信息专辑2023年第一期（总第228期）.pdf", "source": "省交通工程管理中心", "date": "2023-02-17"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=37ebc26c180c4fa3a4d3d29b24d3600f.pdf", "name": "《质监与造价》价格信息专辑2023年第五期（总第232期）.pdf", "source": "省交通工程管理中心", "date": "2023-06-16"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=f95443c3fbf44148963e719b2767c2d7.pdf", "name": "《质监与造价》价格信息专辑2023年第四期（总第231期）.pdf", "source": "省交通工程管理中心", "date": "2023-05-18"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=9e02544c7a444ba6bc0da5d285b560ed.pdf", "name": "《质监与造价》价格信息专辑2023年第二期（总第229期）.pdf", "source": "省交通工程管理中心", "date": "2023-03-20"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=1d40420b6f9c468d94cb3f1d0eb577af.pdf", "name": "《质监与造价》价格信息专辑2021年第七期（总第210期）.pdf", "source": "省交通工程管理中心", "date": "2021-08-13"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=1fe1d4dca46941bf8c9da241e5269861.pdf", "name": "《质监与造价》价格信息专辑2021年第六期（总第209期）.pdf", "source": "省交通工程管理中心", "date": "2021-07-14"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=7c592f33ed434085b60f68b67dd599f0.pdf", "name": "《质监与造价》价格信息专辑2021年第十一期（总第214期）.pdf", "source": "省交通工程管理中心", "date": "2021-12-14"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=30d17daef28547a69bafd96270639e50.pdf", "name": "《质监与造价》价格信息专辑2021年第十二期（总第215期）.pdf", "source": "省交通工程管理中心", "date": "2022-01-11"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=52b5a0bdea234bbba1c0499ef2a454f5.pdf", "name": "《质监与造价》价格信息专辑2021年第八期（总第211期）.pdf", "source": "省交通工程管理中心", "date": "2021-09-15"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=3bf36f92107a4deb8bbae0ff26acf9d9.pdf", "name": "《质监与造价》价格信息专辑2021年第五期（总第208期）.pdf", "source": "省交通工程管理中心", "date": "2021-06-17"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=b1674e6ad80d4e62b7cf3db3a250e30f.pdf", "name": "《质监与造价》价格信息专辑2021年第九期（总第212期）.pdf", "source": "省交通工程管理中心", "date": "2021-10-14"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=f5305a086b1b448383225db31b6ae0a6.pdf", "name": "《质监与造价》价格信息专辑2021年第十期（总第213期）.pdf", "source": "省交通工程管理中心", "date": "2021-11-12"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=63171cac60d446888953f44f24aa201e.pdf", "name": "《质监与造价》价格信息专辑2023年第八期（总第235期）.pdf", "source": "省交通工程管理中心", "date": "2023-09-14"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=e8a1565edb4e4fe4823c23db415e3996.pdf", "name": "《质监与造价》价格信息专辑2023年第九期（总第236期）.pdf", "source": "省交通工程管理中心", "date": "2023-10-18"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=2c98c3aa8c57479c8c8f3e53ed0fd2c3.pdf", "name": "《质监与造价》价格信息专辑2022年第一期（总第216期）.pdf", "source": "省交通工程管理中心", "date": "2022-02-15"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=b05c996abbec421aa0c48c4fe7d52d94.pdf", "name": "《质监与造价》价格信息专辑2023年第六期（总第233期）.pdf", "source": "省交通工程管理中心", "date": "2023-07-14"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=94a4858eed7b49abb0ba714268683a03.pdf", "name": "《质监与造价》价格信息专辑2023年第十一期（总第238期）.pdf", "source": "省交通工程管理中心", "date": "2023-12-15"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=61e5ac8a0d5c431a90667114b6b59323.pdf", "name": "《质监与造价》价格信息专辑2023年第十期（总第237期）.pdf", "source": "省交通工程管理中心", "date": "2023-11-20"},
+{"url": "https://jtyst.zj.gov.cn/module/download/downfile.jsp?classid=0&filename=1767e7d015db40c1863ddccef0aa868e.pdf", "name": "《质监与造价》价格信息专辑2023年第七期（总第234期）.pdf", "source": "省交通工程管理中心", "date": "2023-08-17"}
+]

web/core/extensions.py

@@ -0,0 +1,32 @@
+from concurrent.futures import ThreadPoolExecutor
+
+# from elasticsearch import Elasticsearch
+# from flasgger import Swagger
+# from flask_celery import Celery
+# from flask_cors import CORS
+from flask_sqlalchemy import SQLAlchemy
+from flask_siwadoc import SiwaDoc
+
+import config
+
+db = SQLAlchemy()
+
+# swagger = Swagger(template={
+#     "securityDefinitions": {
+#         "Bearer": {
+#             "type": "apiKey",
+#             "name": "Authorization",
+#             "in": "header",
+#         }
+#     },
+# })
+
+# executor = ThreadPoolExecutor(4)
+
+# es = Elasticsearch(config.ES_HOSTS.split(';'), retry_on_timeout=True)
+
+# celery = Celery()
+
+# cors = CORS()
+
+siwa = SiwaDoc(title="evidence_api", description="evidence_api")

web/core/factory.py

@@ -0,0 +1,71 @@
+from datetime import datetime, date
+from json import JSONEncoder
+
+from flask import Flask
+
+""" 程序入口"""
+
+
+class Singleton(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
+        return cls._instances[cls]
+
+
+class App(Flask):
+    __metaclass__ = Singleton
+
+    def __init__(self, import_name=__name__, **kwargs):
+        super().__init__(import_name, **kwargs)
+        self.bootstrap()
+
+    def bootstrap(self):
+        pass
+
+
+class CustomJSONEncoder(JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, datetime):
+            return obj.strftime('%Y-%m-%d %H:%M:%S')
+        elif isinstance(obj, date):
+            return obj.strftime('%Y-%m-%d')
+        else:
+            return JSONEncoder.default(self, obj)
+
+
+class ApiApp(App):
+    def bootstrap(self):
+        with self.app_context():
+            import config
+            import api
+            from core.extensions import db, siwa
+            self.config.from_object(config)
+            # if config.SENTRY_DSN:
+            #     sentry.init_app(self)
+
+            # if config.CELERY_BROKER_URL:
+            #     celery.init_app(self)
+
+            db.init_app(self)
+
+            siwa.init_app(self)
+
+            api.init_logger(self)
+            api.init_app(self)
+
+            self.json_encoder = CustomJSONEncoder
+
+
+class ClientApp(App):
+    def bootstrap(self):
+        super().bootstrap()
+        with self.app_context():
+            import config
+            from core.extensions import db
+
+            self.config.from_object(config)
+
+            db.init_app(self)

web/core/oil.json

@@ -0,0 +1,232 @@
+[
+{"name": "车用89号汽油（ⅥB）", "price": 9435, "date": "2023-12-19"},
+{"name": "车用92号汽油（ⅥB）", "price": 10001, "date": "2023-12-19"},
+{"name": "车用95号汽油（ⅥB）", "price": 10567, "date": "2023-12-19"},
+{"name": "车用0号柴油（Ⅵ）", "price": 8400, "date": "2023-12-19"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 8904, "date": "2023-12-19"},
+{"name": "车用89号汽油（ⅥB）", "price": 9435, "date": "2023-12-19"},
+{"name": "车用92号汽油（ⅥB）", "price": 10001, "date": "2023-12-19"},
+{"name": "车用95号汽油（ⅥB）", "price": 10567, "date": "2023-12-19"},
+{"name": "车用0号柴油（Ⅵ）", "price": 8400, "date": "2023-12-19"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 8904, "date": "2023-12-19"},
+{"name": "车用89号汽油（ⅥB）", "price": "9585", "date": "2023-07-12"},
+{"name": "车用92号汽油（ⅥB）", "price": "10160", "date": "2023-07-12"},
+{"name": "车用95号汽油（ⅥB）", "price": "10735", "date": "2023-07-12"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8550", "date": "2023-07-12"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "9063", "date": "2023-07-12"},
+{"name": "车用89号汽油（ⅥB）", "price": "9585", "date": "2023-07-12"},
+{"name": "车用92号汽油（ⅥB）", "price": "10160", "date": "2023-07-12"},
+{"name": "车用89号汽油（ⅥB）", "price": 10245, "date": "2023-11-07"},
+{"name": "车用92号汽油（ⅥB）", "price": 10860, "date": "2023-11-07"},
+{"name": "车用89号汽油（ⅥB）", "price": 9635, "date": "2024-01-03"},
+{"name": "车用92号汽油（ⅥB）", "price": 10213, "date": "2024-01-03"},
+{"name": "车用95号汽油（ⅥB）", "price": 10791, "date": "2024-01-03"},
+{"name": "车用0号柴油（Ⅵ）", "price": 8590, "date": "2024-01-03"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 9105, "date": "2024-01-03"},
+{"name": "车用89号汽油（ⅥB）", "price": "9430", "date": "2023-06-28"},
+{"name": "车用92号汽油（ⅥB）", "price": "9996", "date": "2023-06-28"},
+{"name": "车用95号汽油（ⅥB）", "price": "10562", "date": "2023-06-28"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8400", "date": "2023-06-28"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "8904", "date": "2023-06-28"},
+{"name": "车用89号汽油（ⅥB）", "price": "9430", "date": "2023-06-28"},
+{"name": "车用89号汽油（ⅥB）", "price": 9850, "date": "2023-12-05"},
+{"name": "车用92号汽油（ⅥB）", "price": 10441, "date": "2023-12-05"},
+{"name": "车用95号汽油（ⅥB）", "price": 11032, "date": "2023-12-05"},
+{"name": "车用0号柴油（Ⅵ）", "price": 8800, "date": "2023-12-05"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 9328, "date": "2023-12-05"},
+{"name": "车用89号汽油（ⅥB）", "price": 9850, "date": "2023-12-05"},
+{"name": "车用89号汽油（ⅥB）", "price": 10385, "date": "2023-10-24"},
+{"name": "车用92号汽油（ⅥB）", "price": 11008, "date": "2023-10-24"},
+{"name": "车用95号汽油（ⅥB）", "price": 11631, "date": "2023-10-24"},
+{"name": "车用0号柴油（Ⅵ）", "price": 9315, "date": "2023-10-24"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 9874, "date": "2023-10-24"},
+{"name": "车用89号汽油（ⅥB）", "price": 10385, "date": "2023-10-24"},
+{"name": "车用95号汽油（ⅥB）", "price": "10735", "date": "2023-07-12"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8550", "date": "2023-07-12"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "9063", "date": "2023-07-12"},
+{"name": "车用89号汽油（ⅥB）", "price": 9905, "date": "2023-11-21"},
+{"name": "车用92号汽油（ⅥB）", "price": 10499, "date": "2023-11-21"},
+{"name": "车用95号汽油（ⅥB）", "price": 11094, "date": "2023-11-21"},
+{"name": "车用0号柴油（Ⅵ）", "price": 8850, "date": "2023-11-21"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 9381, "date": "2023-11-21"},
+{"name": "车用89号汽油（ⅥB）", "price": 9905, "date": "2023-11-21"},
+{"name": "车用92号汽油（ⅥB）", "price": 10499, "date": "2023-11-21"},
+{"name": "车用95号汽油（ⅥB）", "price": 11474, "date": "2023-11-07"},
+{"name": "车用0号柴油（Ⅵ）", "price": 9180, "date": "2023-11-07"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 9731, "date": "2023-11-07"},
+{"name": "车用89号汽油（ⅥB）", "price": 10245, "date": "2023-11-07"},
+{"name": "车用92号汽油（ⅥB）", "price": 10860, "date": "2023-11-07"},
+{"name": "车用95号汽油（ⅥB）", "price": 11474, "date": "2023-11-07"},
+{"name": "车用0号柴油（Ⅵ）", "price": 9180, "date": "2023-11-07"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 9731, "date": "2023-11-07"},
+{"name": "车用89号汽油（ⅥB）", "price": 9635, "date": "2024-01-03"},
+{"name": "车用92号汽油（ⅥB）", "price": 10213, "date": "2024-01-03"},
+{"name": "车用95号汽油（ⅥB）", "price": 10791, "date": "2024-01-03"},
+{"name": "车用0号柴油（Ⅵ）", "price": 8590, "date": "2024-01-03"},
+{"name": "车用89号汽油（ⅥB）", "price": 9860, "date": "2023-07-26"},
+{"name": "车用92号汽油（ⅥB）", "price": 10452, "date": "2023-07-26"},
+{"name": "车用95号汽油（ⅥB）", "price": 11043, "date": "2023-07-26"},
+{"name": "车用0号柴油（Ⅵ）", "price": 8810, "date": "2023-07-26"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 9339, "date": "2023-07-26"},
+{"name": "车用89号汽油（ⅥB）", "price": 9860, "date": "2023-07-26"},
+{"name": "车用92号汽油（ⅥB）", "price": "9996", "date": "2023-06-28"},
+{"name": "车用95号汽油（ⅥB）", "price": "10562", "date": "2023-06-28"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8400", "date": "2023-06-28"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "8904", "date": "2023-06-28"},
+{"name": "车用92号汽油（ⅥB）", "price": 10441, "date": "2023-12-05"},
+{"name": "车用95号汽油（ⅥB）", "price": 11032, "date": "2023-12-05"},
+{"name": "车用0号柴油（Ⅵ）", "price": 8800, "date": "2023-12-05"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 9328, "date": "2023-12-05"},
+{"name": "车用89号汽油（ⅥB）", "price": 10100, "date": "2023-08-09"},
+{"name": "车用92号汽油（ⅥB）", "price": 10706, "date": "2023-08-09"},
+{"name": "车用95号汽油（ⅥB）", "price": 11312, "date": "2023-08-09"},
+{"name": "车用0号柴油（Ⅵ）", "price": 9040, "date": "2023-08-09"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 9582, "date": "2023-08-09"},
+{"name": "车用89号汽油（ⅥB）", "price": 10100, "date": "2023-08-09"},
+{"name": "车用92号汽油（ⅥB）", "price": 10706, "date": "2023-08-09"},
+{"name": "车用95号汽油（ⅥB）", "price": 11312, "date": "2023-08-09"},
+{"name": "车用92号汽油（ⅥB）", "price": 11008, "date": "2023-10-24"},
+{"name": "车用95号汽油（ⅥB）", "price": 11631, "date": "2023-10-24"},
+{"name": "车用0号柴油（Ⅵ）", "price": 9315, "date": "2023-10-24"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 9874, "date": "2023-10-24"},
+{"name": "车用89号汽油（ⅥB）", "price": 10155, "date": "2023-08-23"},
+{"name": "车用92号汽油（ⅥB）", "price": 10764, "date": "2023-08-23"},
+{"name": "车用95号汽油（ⅥB）", "price": 11374, "date": "2023-08-23"},
+{"name": "车用0号柴油（Ⅵ）", "price": 9095, "date": "2023-08-23"},
+{"name": "车用89号汽油（ⅥB）", "price": 10540, "date": "2023-09-20"},
+{"name": "车用92号汽油（ⅥB）", "price": 11172, "date": "2023-09-20"},
+{"name": "车用95号汽油（ⅥB）", "price": 11805, "date": "2023-09-20"},
+{"name": "车用0号柴油（Ⅵ）", "price": 9465, "date": "2023-09-20"},
+{"name": "车用89号汽油（ⅥB）", "price": "10455", "date": "2023-10-10"},
+{"name": "车用92号汽油（ⅥB）", "price": "11082", "date": "2023-10-10"},
+{"name": "车用95号汽油（ⅥB）", "price": "11710", "date": "2023-10-10"},
+{"name": "车用0号柴油（Ⅵ）", "price": "9385", "date": "2023-10-10"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "9948", "date": "2023-10-10"},
+{"name": "车用89号汽油（ⅥB）", "price": "10455", "date": "2023-10-10"},
+{"name": "车用95号汽油（ⅥB）", "price": 11094, "date": "2023-11-21"},
+{"name": "车用0号柴油（Ⅵ）", "price": 8850, "date": "2023-11-21"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 9381, "date": "2023-11-21"},
+{"name": "车用89号汽油（ⅥB）", "price": "9855", "date": "2023-04-17"},
+{"name": "车用92号汽油（ⅥB）", "price": "10446", "date": "2023-04-17"},
+{"name": "车用95号汽油（ⅥB）", "price": "11038", "date": "2023-04-17"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8805", "date": "2023-04-17"},
+{"name": "车用89号汽油（ⅥB）", "price": "9305", "date": "2023-03-31"},
+{"name": "车用92号汽油（ⅥB）", "price": "9863", "date": "2023-03-31"},
+{"name": "车用95号汽油（ⅥB）", "price": "10422", "date": "2023-03-31"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8280", "date": "2023-03-31"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "8777", "date": "2023-03-31"},
+{"name": "车用89号汽油（ⅥA）", "price": "9735", "date": "2023-01-03"},
+{"name": "车用92号汽油（ⅥA）", "price": "10319", "date": "2023-01-03"},
+{"name": "车用95号汽油（ⅥA）", "price": "10903", "date": "2023-01-03"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8690", "date": "2023-01-03"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "9211", "date": "2023-01-03"},
+{"name": "车用89号汽油（ⅥA）", "price": "9735", "date": "2023-01-03"},
+{"name": "车用89号汽油（ⅥA）", "price": "9640", "date": "2023-03-17"},
+{"name": "车用92号汽油（ⅥA）", "price": "10218", "date": "2023-03-17"},
+{"name": "车用95号汽油（ⅥA）", "price": "10797", "date": "2023-03-17"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8600", "date": "2023-03-17"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "9116", "date": "2023-03-17"},
+{"name": "车用89号汽油（ⅥA）", "price": "9640", "date": "2023-03-17"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 9105, "date": "2024-01-03"},
+{"name": "车用89号汽油（ⅥA）", "price": "9530", "date": "2023-01-17"},
+{"name": "车用92号汽油（ⅥA）", "price": "10102", "date": "2023-01-17"},
+{"name": "车用95号汽油（ⅥA）", "price": "10674", "date": "2023-01-17"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8495", "date": "2023-01-17"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "9005", "date": "2023-01-17"},
+{"name": "车用92号汽油（ⅥB）", "price": 10452, "date": "2023-07-26"},
+{"name": "车用95号汽油（ⅥB）", "price": 11043, "date": "2023-07-26"},
+{"name": "车用0号柴油（Ⅵ）", "price": 8810, "date": "2023-07-26"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 9339, "date": "2023-07-26"},
+{"name": "车用89号汽油（ⅥB）", "price": "9415", "date": "2023-05-30"},
+{"name": "车用92号汽油（ⅥB）", "price": "9980", "date": "2023-05-30"},
+{"name": "车用95号汽油（ⅥB）", "price": "10545", "date": "2023-05-30"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8380", "date": "2023-05-30"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "8883", "date": "2023-05-30"},
+{"name": "车用89号汽油（ⅥA）", "price": "9740", "date": "2023-02-03"},
+{"name": "车用92号汽油（ⅥA）", "price": "10324", "date": "2023-02-03"},
+{"name": "车用95号汽油（ⅥA）", "price": "10909", "date": "2023-02-03"},
+{"name": "车用89号汽油（ⅥB）", "price": "9695", "date": "2023-04-28"},
+{"name": "车用92号汽油（ⅥB）", "price": "10277", "date": "2023-04-28"},
+{"name": "车用95号汽油（ⅥB）", "price": "10858", "date": "2023-04-28"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8650", "date": "2023-04-28"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "9169", "date": "2023-04-28"},
+{"name": "车用89号汽油（ⅥB）", "price": "9695", "date": "2023-04-28"},
+{"name": "车用89号汽油（ⅥB）", "price": "9360", "date": "2023-06-13"},
+{"name": "车用92号汽油（ⅥB）", "price": "9922", "date": "2023-06-13"},
+{"name": "车用95号汽油（ⅥB）", "price": "10483", "date": "2023-06-13"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8330", "date": "2023-06-13"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "8830", "date": "2023-06-13"},
+{"name": "车用89号汽油（ⅥB）", "price": "9360", "date": "2023-06-13"},
+{"name": "车用89号汽油（ⅥB）", "price": "9315", "date": "2023-05-16"},
+{"name": "车用92号汽油（ⅥB）", "price": "9874", "date": "2023-05-16"},
+{"name": "车用95号汽油（ⅥB）", "price": "10433", "date": "2023-05-16"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8285", "date": "2023-05-16"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "8782", "date": "2023-05-16"},
+{"name": "车用89号汽油（ⅥB）", "price": "9315", "date": "2023-05-16"},
+{"name": "车用0号柴油（Ⅵ）", "price": 9040, "date": "2023-08-09"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 9582, "date": "2023-08-09"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 9641, "date": "2023-08-23"},
+{"name": "车用89号汽油（ⅥB）", "price": 10155, "date": "2023-08-23"},
+{"name": "车用92号汽油（ⅥB）", "price": 10764, "date": "2023-08-23"},
+{"name": "车用95号汽油（ⅥB）", "price": 11374, "date": "2023-08-23"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 10033, "date": "2023-09-20"},
+{"name": "车用89号汽油（ⅥB）", "price": 10540, "date": "2023-09-20"},
+{"name": "车用92号汽油（ⅥB）", "price": 11172, "date": "2023-09-20"},
+{"name": "车用95号汽油（ⅥB）", "price": 11805, "date": "2023-09-20"},
+{"name": "车用0号柴油（Ⅵ）", "price": 9465, "date": "2023-09-20"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 10033, "date": "2023-09-20"},
+{"name": "车用92号汽油（ⅥB）", "price": "11082", "date": "2023-10-10"},
+{"name": "车用95号汽油（ⅥB）", "price": "11710", "date": "2023-10-10"},
+{"name": "车用0号柴油（Ⅵ）", "price": "9385", "date": "2023-10-10"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "9948", "date": "2023-10-10"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "9333", "date": "2023-04-17"},
+{"name": "车用89号汽油（ⅥB）", "price": "9855", "date": "2023-04-17"},
+{"name": "车用92号汽油（ⅥB）", "price": "10446", "date": "2023-04-17"},
+{"name": "车用95号汽油（ⅥB）", "price": "11038", "date": "2023-04-17"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8805", "date": "2023-04-17"},
+{"name": "车用89号汽油（ⅥB）", "price": "9305", "date": "2023-03-31"},
+{"name": "车用92号汽油（ⅥB）", "price": "9863", "date": "2023-03-31"},
+{"name": "车用95号汽油（ⅥB）", "price": "10422", "date": "2023-03-31"},
+{"name": "车用92号汽油（ⅥA）", "price": "10319", "date": "2023-01-03"},
+{"name": "车用95号汽油（ⅥA）", "price": "10903", "date": "2023-01-03"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8690", "date": "2023-01-03"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "9211", "date": "2023-01-03"},
+{"name": "车用92号汽油（ⅥA）", "price": "10218", "date": "2023-03-17"},
+{"name": "车用95号汽油（ⅥA）", "price": "10797", "date": "2023-03-17"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8600", "date": "2023-03-17"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "9116", "date": "2023-03-17"},
+{"name": "车用89号汽油（ⅥA）", "price": "9530", "date": "2023-01-17"},
+{"name": "车用92号汽油（ⅥA）", "price": "10102", "date": "2023-01-17"},
+{"name": "车用95号汽油（ⅥA）", "price": "10674", "date": "2023-01-17"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8495", "date": "2023-01-17"},
+{"name": "车用89号汽油（ⅥB）", "price": "9415", "date": "2023-05-30"},
+{"name": "车用92号汽油（ⅥB）", "price": "9980", "date": "2023-05-30"},
+{"name": "车用95号汽油（ⅥB）", "price": "10545", "date": "2023-05-30"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8380", "date": "2023-05-30"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "8883", "date": "2023-05-30"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8695", "date": "2023-02-03"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "9217", "date": "2023-02-03"},
+{"name": "车用89号汽油（ⅥA）", "price": "9740", "date": "2023-02-03"},
+{"name": "车用92号汽油（ⅥB）", "price": "10277", "date": "2023-04-28"},
+{"name": "车用95号汽油（ⅥB）", "price": "10858", "date": "2023-04-28"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8650", "date": "2023-04-28"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "9169", "date": "2023-04-28"},
+{"name": "车用92号汽油（ⅥB）", "price": "9922", "date": "2023-06-13"},
+{"name": "车用95号汽油（ⅥB）", "price": "10483", "date": "2023-06-13"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8330", "date": "2023-06-13"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "8830", "date": "2023-06-13"},
+{"name": "车用92号汽油（ⅥB）", "price": "9874", "date": "2023-05-16"},
+{"name": "车用95号汽油（ⅥB）", "price": "10433", "date": "2023-05-16"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8285", "date": "2023-05-16"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "8782", "date": "2023-05-16"},
+{"name": "车用0号柴油（Ⅵ）", "price": 9095, "date": "2023-08-23"},
+{"name": "车用-10号柴油（Ⅵ）", "price": 9641, "date": "2023-08-23"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "9333", "date": "2023-04-17"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8280", "date": "2023-03-31"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "8777", "date": "2023-03-31"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "9005", "date": "2023-01-17"},
+{"name": "车用92号汽油（ⅥA）", "price": "10324", "date": "2023-02-03"},
+{"name": "车用95号汽油（ⅥA）", "price": "10909", "date": "2023-02-03"},
+{"name": "车用0号柴油（Ⅵ）", "price": "8695", "date": "2023-02-03"},
+{"name": "车用-10号柴油（Ⅵ）", "price": "9217", "date": "2023-02-03"}
+]

web/gunicorn_conf.py

@@ -0,0 +1,39 @@
+import os
+
+from worker import ASYNC_MODE
+from config import DEBUG, PORT
+
+bind = '0.0.0.0:{}'.format(PORT)
+backlog = 2048
+
+if ASYNC_MODE == 'gevent':
+    worker_class = 'gevent'
+
+workers = os.cpu_count() if os.cpu_count() <= 4 else int(os.cpu_count() / 4) + 1
+preload_app = True
+
+debug = False
+spew = False
+
+timeout = 600
+
+proc_name = 'app'
+
+limit_request_line = 0  # 4094
+
+# debug
+if DEBUG:
+    loglevel = 'debug'
+    graceful_timeout = 1
+    reload = True
+    # capture_output = True
+
+else:
+    loglevel = 'info'
+    # errorlog = './gunicorn-error.log'
+    # accesslog = './gunicorn-access.log'
+
+try:
+    from local_gunicorn_conf import *  # noqa
+except ImportError:
+    pass

web/run.bat

@@ -0,0 +1,2 @@
+title server
+python36 server_waitress.py

web/run_celery.bat

@@ -0,0 +1,2 @@
+title celery
+python36 -m celery -A tasks.asynced.celery_worker.celery worker -c 4 --loglevel=INFO --pool=eventlet

web/run_cron.bat

@@ -0,0 +1,3 @@
+title cron
+python36 -m tasks.crond.scheduler
+

web/server_waitress.py

@@ -0,0 +1,14 @@
+from os import cpu_count
+
+from waitress import serve
+
+from core.app import app
+from core.conf import PORT
+
+if __name__ == '__main__':
+    serve(
+        app,
+        host='0.0.0.0',
+        port=PORT,
+        threads=int(cpu_count())  # int(2 * cpu_count() + 1)
+    )

web/spiders/__init__.py

@@ -0,0 +1,157 @@
+import json
+import logging
+import pathlib
+from urllib.parse import unquote
+
+import requests
+from lxml import etree
+from scrapy.crawler import CrawlerProcess
+from scrapy.utils.project import get_project_settings
+
+from utils.login import login_mysteel, login_baiinfo
+
+
+def run_spider(spider):
+    filename = f'{spider.name}.json'
+    settings = get_project_settings()
+    settings.set('FEEDS', {
+        f'{filename}': {
+            'format': 'json',
+            'encoding': 'utf8',
+            'overwrite': True,
+        },
+    })
+    process = CrawlerProcess(settings)
+    process.crawl(spider)
+    process.start()
+    return pathlib.Path(filename)  # .absolute()
+
+class CookieTools:
+    user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36'
+
+
+class MysteelCookieTools(CookieTools):
+
+    @staticmethod
+    def is_valid(cookies):
+        result = requests.get('https://e.mysteel.com/account_index.htm', cookies=cookies)
+        result.encoding = result.apparent_encoding
+        tree = etree.HTML(result.text)
+
+        flag_tag = tree.xpath('/html/body/div[1]/div[1]/p[2]/text()')
+        print(flag_tag)
+        if len(flag_tag) > 0 and flag_tag[0] == '会员登录':
+            logging.warning('Mysteel.com Cookies 无效或已过期 | 强制跳转至登陆页')
+            return False
+
+        flag_tag = tree.xpath('//*[@id="top"]/div/span[2]/a/text()')
+        print(flag_tag)
+        if len(flag_tag) > 0 and flag_tag[0] == '退出登录':
+            pass
+        else:
+            logging.warning('Mysteel.com Cookies 无效或已过期 | 无法正确进入个人中心')
+            return False
+
+        logging.warning('Mysteel.com Cookies 验证成功 | 成功进入个人中心')
+        return True
+
+    @staticmethod
+    def save_as_json(raw_cookies, file_path):
+        try:
+            login_token = [i for i in raw_cookies if i['name'] == '_login_token'][0].get('value')
+            session = [i for i in raw_cookies if i['name'] == '_MSPASS_SESSION'][0].get('value')
+            text = json.dumps({
+                '_login_token': login_token,
+                login_token: '1=10',
+                '_MSPASS_SESSION': session
+            })
+            with open(file_path, 'w') as f:
+                f.write(text)
+            return file_path
+        except IndexError:
+            logging.warning('保存失败 | 无法正确解析原始 cookies')
+
+    @staticmethod
+    def read_from_json(file_path):
+        cookies = json.loads(open(file_path, 'r', encoding='utf-8').read())
+        return cookies
+
+    @classmethod
+    def get_cookies(cls, file_path=r'E:\Project\item_spider\mysteel.cookies.json'):
+        cookie_json_file_path = file_path
+        cookie = cls.read_from_json(cookie_json_file_path)
+        if not cls.is_valid(cookie):
+            raw_cookies = login_mysteel()
+            cls.save_as_json(raw_cookies, cookie_json_file_path)
+            cookie = cls.read_from_json(cookie_json_file_path)
+        return cookie
+
+
+class BaiinfoCookieTools(CookieTools):
+    @staticmethod
+    def is_valid(cookies):
+        if not cookies:
+            return False
+        result = requests.post(
+            'http://www.baiinfo.com/api/website/price/priceInfo/getPriceList',
+            json={"channelId": "18", "pricesGroupId": 526},
+            # cookies=cookies,
+            headers={
+                'Baiinfo-Auth': json.loads(cookies['user'])['token'],
+                'User-Agent': CookieTools.user_agent
+            }
+        )
+        flag = json.loads(result.text)
+        if flag['code'] != 200:
+            logging.warning(f'Baiinfo.com Token 无效或已过期 | {flag["msg"]}')
+            return False
+
+        result = requests.get('http://www.baiinfo.com/news/newscategory/4710/99/3', cookies=cookies, headers={'User-Agent': 'PostmanRuntime/7.26.8'})
+        # result.encoding = result.apparent_encoding
+        tree = etree.HTML(result.text)
+
+        flag_tag = tree.xpath('//head/title/text()')
+        print(flag_tag)
+        if len(flag_tag) > 0 and '用户登录' in flag_tag[0]:
+            logging.warning('Baiinfo.com Cookies 无效或已过期 | 强制跳转至登陆页')
+            return False
+
+        flag_tag = tree.xpath('//head/title/text()')
+        print(flag_tag)
+        if len(flag_tag) > 0 and '水泥价格(华东) - 百川盈孚' in flag_tag[0]:
+            pass
+        else:
+            logging.warning('Baiinfo.com Cookies 无效或已过期 | 无法正确进入鉴权页面')
+            return False
+
+        logging.warning('Baiinfo.com Cookies 验证成功 | 成功进入个鉴权页面')
+        return True
+
+    @staticmethod
+    def read_from_json(file_path):
+        raw_cookies = json.loads(open(file_path, 'r', encoding='utf-8').read() or '[]')
+        user_cookie = [i for i in raw_cookies if i['name'] == 'user']
+        if not user_cookie:
+            return None
+        user = user_cookie[0].get('value')
+        user = unquote(user)
+        return {'user': user}
+
+    @staticmethod
+    def save_as_json(raw_cookies, file_path):
+        try:
+            with open(file_path, 'w') as f:
+                f.write(json.dumps(raw_cookies))
+            return file_path
+        except IndexError:
+            logging.warning('保存失败 | 无法正确解析原始 cookies')
+
+    @classmethod
+    def get_cookies(cls, file_path=r'E:\Project\item_spider\baiinfo.cookies.json'):
+        cookie_json_file_path = file_path
+        cookie = cls.read_from_json(cookie_json_file_path)
+        if not cls.is_valid(cookie):
+            raw_cookies = login_baiinfo()
+            cls.save_as_json(raw_cookies, cookie_json_file_path)
+            cookie = cls.read_from_json(cookie_json_file_path)
+        return cookie

web/spiders/asphalt_domestic.py

@@ -0,0 +1,79 @@
+import scrapy
+from scrapy import Request
+
+
+
+class AsphaltDomesticSpider(scrapy.Spider):
+    name = "asphalt_domestic"
+    start_urls = [
+        ('国内炼厂重交沥青出厂价格', "http://www.baiinfo.com/api/website/price/priceInfo/getPriceList",
+         {"channelId": "18", "pricesGroupId": 526}),
+        ('国内市场沥青批发价格汇总', "http://www.baiinfo.com/api/website/price/priceInfo/getPriceList",
+         {"channelId": "18", "pricesGroupId": 530}),
+    ]
+    cookie = None
+    user_agent = None
+    _token = None
+
+    @property
+    def token(self):
+        if self._token:
+            return self._token
+        else:
+            self._token = json.loads(cookie['user'])['token']
+            return self._token
+
+    def start_requests(self):
+        for source, url, data in self.start_urls:
+            yield Request(
+                method='POST',
+                body=json.dumps(data),
+                url=url,
+                headers={
+                    'User-Agent': self.user_agent,
+                    'Content-Type': 'application/json',
+                    'Baiinfo-Auth': self.token,
+                    # 'Baiinfo-Auth': TOKEN,
+                },
+                meta={'source': source}
+            )
+
+    def parse(self, response, **kwargs):
+        ret = json.loads(response.text)
+        if ret.get('success') and ret.get('data'):
+            for item in ret['data']:
+                item['source'] = response.meta['source']
+                for date in item['priceData']:
+                    try:
+                        price = int(item['priceData'][date])
+                    except ValueError:
+                        price = 0
+                    yield {
+                        'name': item['targetName'],
+                        'price': price,
+                        'date': date,
+                        # 'fluctuating': item['changePriceData'][date],
+                        'from_': response.meta['source'],
+                    }
+
+
+if __name__ == '__main__':
+    import json
+
+    from spiders import run_spider, BaiinfoCookieTools
+    from commons.models.asphalt_domestic import AsphaltDomestic
+    from core.factory import ClientApp
+
+    # cookie 读取
+    cookie = BaiinfoCookieTools.get_cookies()
+    # 爬取
+    AsphaltDomesticSpider.cookie = cookie
+    AsphaltDomesticSpider.user_agent = BaiinfoCookieTools.user_agent
+    file_path = run_spider(AsphaltDomesticSpider)
+    # 入库
+    data = json.loads(open(file_path, 'r', encoding='utf-8').read())
+    with ClientApp().app_context():
+        for item in data:
+            print(item)
+            AsphaltDomestic(**item).upsert()
+

web/spiders/asphalt_imported.py

@@ -0,0 +1,85 @@
+import scrapy
+from scrapy import Request
+
+
+class AsphaltImportedSpider(scrapy.Spider):
+    name = "asphalt_imported"
+    start_urls = [
+        ('沥青', "http://www.baiinfo.com/news/newscategory/17847/18/1"),
+        ('沥青', "http://www.baiinfo.com/news/newscategory/17847/18/2"),
+        ('沥青', "http://www.baiinfo.com/news/newscategory/17847/18/3"),
+        ('沥青', "http://www.baiinfo.com/news/newscategory/17847/18/4"),
+        ('沥青', "http://www.baiinfo.com/news/newscategory/17847/18/5"),
+    ]
+    cookie = None
+    user_agent = None
+
+    def start_requests(self):
+        for source, url in self.start_urls:
+            yield Request(
+                method='GET',
+                url=url,
+                headers={
+                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
+                },
+                cookies=self.cookie,
+                meta={'source': source, 'type': 'home'}
+            )
+
+    def parse(self, response, **kwargs):
+        if response.meta['type'] == 'home':
+            yield from self.parse_home(response)
+        elif response.meta['type'] == 'list_page':
+            yield from self.parse_list_page(response)
+
+    def parse_home(self, response):
+        for item in response.xpath('//*[@id="__nuxt"]/div/div[5]/div/div[2]/div[1]/div[1]/div[2]/ul/li/a'):
+            name = item.xpath('text()').get()
+            if '散装进口沥青到岸价' in name:
+                print(name, 'http://www.baiinfo.com{}'.format(item.xpath('@href').get()))
+                yield Request(
+                    method='GET',
+                    url='http://www.baiinfo.com{}'.format(item.xpath('@href').get()),
+                    headers={
+                        'User-Agent': self.user_agent,
+                    },
+                    cookies=self.cookie,
+                    meta={'source': name, 'type': 'list_page'}
+                )
+
+    def parse_list_page(self, response):
+        date = response.xpath('//*[@id="__nuxt"]/div/div[5]/div/div[2]/div[1]/div[2]/p[1]/span[4]/text()').get()
+        date = date.strip('日').replace('年', '-').replace('月', '-')
+
+        for item in response.xpath('//*[@id="__nuxt"]/div/div[5]/div/div[2]/div[1]/div[3]/div/div/table/tbody/tr'):
+            line = [cell.xpath('text()').get() for cell in item.xpath('td/span/span/span/span/span')][:7]
+            print(line)
+            if line[-1] == '备注' or '品质' in line[0]:
+                continue
+            name, *_, price, fluctuating = line
+            yield {
+                'name': name,
+                'date': date,
+                'price': int(price.split('-')[-1]),
+            }
+
+
+if __name__ == '__main__':
+    import json
+
+    from spiders import run_spider, BaiinfoCookieTools
+    from commons.models.asphalt_imported import AsphaltImported
+    from core.factory import ClientApp
+
+    # cookie 读取
+    cookie = BaiinfoCookieTools.get_cookies()
+    # 爬取
+    AsphaltImportedSpider.cookie = cookie
+    AsphaltImportedSpider.user_agent = BaiinfoCookieTools.user_agent
+    file_path = run_spider(AsphaltImportedSpider)
+    # 入库
+    data = json.loads(open(file_path, 'r', encoding='utf-8').read())
+    with ClientApp().app_context():
+        for item in data:
+            print(item)
+            AsphaltImported(**item).upsert()

web/spiders/cement.py

@@ -0,0 +1,96 @@
+import scrapy
+from scrapy import Request
+
+
+class CementSpider(scrapy.Spider):
+    name = "cement"
+    start_urls = [
+        ('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/1"),
+        ('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/2"),
+        ('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/3"),
+        ('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/4"),
+        ('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/5"),
+        ('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/6"),
+        ('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/7"),
+        ('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/8"),
+        ('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/9"),
+        ('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/10"),
+    ]
+    cookie = None
+    user_agent = None
+
+    def start_requests(self):
+        for source, url in self.start_urls:
+            yield Request(
+                method='GET',
+                url=url,
+                headers={
+                    'User-Agent': self.user_agent,
+                },
+                cookies=self.cookie,
+                meta={'source': source, 'type': 'home'}
+            )
+
+    def parse(self, response, **kwargs):
+        if response.meta['type'] == 'home':
+            yield from self.parse_home(response)
+        elif response.meta['type'] == 'list_page':
+            yield from self.parse_list_page(response)
+
+    def parse_home(self, response):
+        for item in response.xpath('//*[@id="__nuxt"]/div/div[5]/div/div[2]/div[1]/div[1]/div[2]/ul/li/a'):
+            title = item.xpath('text()').get()
+            if '福建水泥市场参考价格' in title:
+                print(title, 'http://www.baiinfo.com{}'.format(item.xpath('@href').get()))
+                yield Request(
+                    method='GET',
+                    url='http://www.baiinfo.com{}'.format(item.xpath('@href').get()),
+                    headers={
+                        'User-Agent': self.user_agent,
+                    },
+                    cookies=self.cookie,
+                    meta={'source': title, 'type': 'list_page'}
+                )
+
+    def parse_list_page(self, response):
+        date = response.xpath('//*[@id="__nuxt"]/div/div[5]/div/div[2]/div[1]/div[2]/p[1]/span[4]/text()').get()
+        date = date.strip('日').replace('年', '-').replace('月', '-')
+
+        for item in response.xpath('//tr[position()>2]'):
+            block_1 = [cell.get() for cell in item.xpath('td/span/span/span/span/span/text()') if cell.get()]
+            price, *_ = block_1
+
+            block_2 = [cell.get() for cell in item.xpath('td/span/span/span/span/span/span/text()') if cell.get()]
+            spec, name, pack, source, _, fluctuating = block_2
+
+            yield {
+                'name': name,
+                'price': price,
+                'spec': spec,
+                'pack': pack,
+                'date': date,
+                'source': source,
+                'fluctuating': int(fluctuating)
+            }
+
+
+if __name__ == '__main__':
+    import json
+
+    from spiders import run_spider, BaiinfoCookieTools
+    from commons.models.cement import Cement
+    from core.factory import ClientApp
+
+    # cookie 读取
+    cookie = BaiinfoCookieTools.get_cookies()
+    # 爬取
+    CementSpider.cookie = cookie
+    CementSpider.user_agent = BaiinfoCookieTools.user_agent
+    file_path = run_spider(CementSpider)
+    # 入库
+    data = json.loads(open(file_path, 'r', encoding='utf-8').read())
+    with ClientApp().app_context():
+        for item in data:
+            print(item)
+            Cement(**item).upsert()
+

web/spiders/data_fujian.py

@@ -0,0 +1,91 @@
+import datetime
+from dateutil.relativedelta import relativedelta
+
+import scrapy
+from scrapy import Request
+
+
+CITY_ID = [
+    ('93', '福州市'),
+    ('94', '厦门市'),
+    ('95', '宁德市'),
+    ('96', '莆田市'),
+    ('97', '泉州市'),
+    ('98', '漳州市'),
+    ('99', '龙岩市'),
+    ('100', '三明市'),
+    ('101', '南平市'),
+    ('102', '平潭综合实验区'),
+]
+
+CLASS_CODE = [
+    ('01', '01黑色及有色金属'),
+    ('04', '04水泥、砖瓦灰砂'),
+    ('05', '05木、竹材料及其制品'),
+    ('13', '13涂料及防腐、防水材料'),
+    ('14', '14油品、化工原料'),
+]
+
+URL = 'http://49.4.85.126/Information/Index?qClassCode={class_code}&qMatType=0&WayID=14&WayID2=4&CityID=7&CityID2={city_id}&Year={year}&Month={month}&Week=0&Day=0&qKeyWord='
+MONTHS = 2  # 爬取最近月份数量
+
+
+class DataFujianSpider(scrapy.Spider):
+    name = "data_fujian"
+
+    def start_requests(self):
+        for city_id, city_name in CITY_ID:
+            for class_code, source in CLASS_CODE:
+                for month in range(1, 1 + MONTHS):
+                    date = datetime.date.today() - relativedelta(months=month)
+                    yield Request(
+                        method='GET',
+                        url=URL.format(year=date.year, month=date.month, class_code=class_code, city_id=city_id),
+                        headers={
+                            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
+                        },
+                        meta={'source': source, 'type': 'home', 'city': city_name, 'month': date.month, 'year': date.year}
+                    )
+
+    def parse(self, response, **kwargs):
+        for item in response.xpath('//*[@id="searcList"]/div/div[2]/div[3]/div[3]/table/tr'):
+            block_1 = [i.xpath('text()').get().strip() for i in item.xpath('td')]
+            print(block_1)
+            if not block_1:
+                continue
+            number, _, type_, unit, price, price_with_tax, *_ = block_1
+
+            block_2 = [i.xpath('text()').get().strip() for i in item.xpath('td/span')]
+            print(block_2)
+            name, *_ = block_2
+
+            yield {
+                'number': number,
+                'name': name,
+                'spec': type_,
+                'unit': unit,
+                'price_without_tax': price,
+                'price': price_with_tax,
+                'category': response.meta['source'],
+                'year': response.meta['year'],
+                'month': response.meta['month'],
+                'city': response.meta['city'],
+                'date': datetime.date.today().strftime('%Y-%m-%d')
+            }
+
+
+if __name__ == '__main__':
+    import json
+
+    from spiders import run_spider
+    from commons.models.data_fujian import DataFujian
+    from core.factory import ClientApp
+
+    # 爬取
+    file_path = run_spider(DataFujianSpider)
+    # 入库
+    data = json.loads(open(file_path, 'r', encoding='utf-8').read())
+    with ClientApp().app_context():
+        for item in data:
+            print(item)
+            DataFujian(**item).upsert()

web/spiders/data_guangdong.py

@@ -0,0 +1,78 @@
+import scrapy
+from scrapy import Request
+
+
+class DataGuangdongSpider(scrapy.Spider):
+    name = "data_guangdong"
+    start_urls = [
+        ('材料信息价', "http://zjz.gdcd.gov.cn/zjzgdcd/zjxx_clxxj/list.shtml"),
+        ('材料信息价', "http://zjz.gdcd.gov.cn/zjzgdcd/zjxx_clxxj/list_2.shtml"),
+        ('材料信息价', "http://zjz.gdcd.gov.cn/zjzgdcd/zjxx_clxxj/list_3.shtml"),
+        ('材料信息价', "http://zjz.gdcd.gov.cn/zjzgdcd/zjxx_clxxj/list_4.shtml"),
+        ('材料信息价', "http://zjz.gdcd.gov.cn/zjzgdcd/zjxx_clxxj/list_5.shtml"),
+    ]
+
+    def start_requests(self):
+        for source, url in self.start_urls:
+            yield Request(
+                method='GET',
+                url=url,
+                headers={
+                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
+                },
+                meta={'source': source, 'type': 'home'}
+            )
+
+    def parse(self, response, **kwargs):
+        if response.meta['type'] == 'home':
+            yield from self.parse_home(response)
+        elif response.meta['type'] == 'list':
+            yield from self.parse_list(response)
+
+    def parse_home(self, response):
+        for item in response.xpath('/html/body/div/div[3]/div[2]/div[2]/div/ul/li/a'):
+            uri = item.xpath('@href').get()
+            name = item.xpath('text()').get()
+            if '广东省交通建设工程主要外购材料信息价表' not in name:
+                continue
+            yield Request(
+                method='GET',
+                url=f'http://zjz.gdcd.gov.cn{uri}',
+                headers={
+                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
+                },
+                meta={'source': name, 'type': 'list'}
+            )
+
+    def parse_list(self, response):
+        date = response.xpath('/html/body/div[1]/div[4]/div/div[1]/div[2]/span[2]/text()').get().strip('发布时间:')
+        source = response.xpath('/html/body/div[1]/div[4]/div/div[1]/div[2]/span[1]/b/text()').get()
+        for item in response.xpath('//*[@id="zoomcon"]/p/a'):
+            uri = item.xpath('@href').get()
+            name = item.xpath('text()').get()
+            url_prefix = '/'.join(response.url.split('/')[:-1])
+            print(uri, name)
+            yield {
+                'url': f'{url_prefix}/{uri}',
+                'name': name,
+                # 'source': response.meta['source'],
+                'source': source,
+                'date': date
+            }
+
+
+if __name__ == '__main__':
+    import json
+
+    from spiders import run_spider
+    from commons.models.data_guangdong import DataGuangdong
+    from core.factory import ClientApp
+
+    # 爬取
+    file_path = run_spider(DataGuangdongSpider)
+    # 入库
+    data = json.loads(open(file_path, 'r', encoding='utf-8').read())
+    with ClientApp().app_context():
+        for item in data:
+            print(item)
+            DataGuangdong(**item).upsert()

web/spiders/date_zhejiang.py

@@ -0,0 +1,84 @@
+import scrapy
+from lxml import html, etree
+from scrapy import Request
+
+
+class DataZhejiangSpider(scrapy.Spider):
+    name = "data_zhejiang"
+    start_urls = [
+        ('材料价格', "http://jtyst.zj.gov.cn/col/col1228999576/index.html"),
+        ('材料价格', "http://jtyst.zj.gov.cn/col/col1228999576/index.html?uid=5509220&pageNum=2"),
+        ('材料价格', "http://jtyst.zj.gov.cn/col/col1228999576/index.html?uid=5509220&pageNum=3"),
+        ('材料价格', "http://jtyst.zj.gov.cn/col/col1228999576/index.html?uid=5509220&pageNum=4"),
+        ('材料价格', "http://jtyst.zj.gov.cn/col/col1228999576/index.html?uid=5509220&pageNum=5"),
+    ]
+
+    def start_requests(self):
+        for source, url in self.start_urls:
+            yield Request(
+                method='GET',
+                url=url,
+                headers={
+                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
+                },
+                meta={'source': source, 'type': 'home'}
+            )
+
+    def parse(self, response, **kwargs):
+        if response.meta['type'] == 'home':
+            yield from self.parse_home(response)
+        elif response.meta['type'] == 'list':
+            yield from self.parse_list(response)
+
+    def parse_home(self, response):
+        rsp = response.xpath('//*[@id="5509220"]/script/text()').get()
+        for t in ('<![CDATA[', ']]>', '</record>', '<record>', '</recordset>', '<recordset>', '</datastore>', '<datastore>', '</nextgroup>', '<nextgroup>'):
+            rsp = rsp.replace(t, '')
+        html = etree.HTML(rsp)
+        for item in html.xpath('//li/a'):
+            print(item)
+            uri = item.xpath('@href')[0]
+            name = item.xpath('text()')[0]
+            print(uri, name)
+            if '《质监与造价》价格信息专辑' not in name:
+                continue
+            yield Request(
+                method='GET',
+                url=f'http://jtyst.zj.gov.cn{uri}',
+                headers={
+                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
+                },
+                meta={'source': name, 'type': 'list'}
+            )
+
+    def parse_list(self, response):
+        date = response.xpath('/html/body/div[2]/div[1]/div[2]/div[1]/p/span[1]/text()').get().split('日')[0].replace('年', '-').replace('月', '-')
+        source = response.xpath('/html/body/div[2]/div[1]/div[2]/div[1]/p/span[2]/text()[2]').get()
+        for item in response.xpath('//*[@id="zoom"]/p/a'):
+            uri = item.xpath('@href').get()
+            name = item.xpath('text()').get()
+            print(uri, name)
+            yield {
+                'url': f'https://jtyst.zj.gov.cn{uri}',
+                'name': name,
+                # 'source': response.meta['source']
+                'source': source,
+                'date': date,
+            }
+
+
+if __name__ == '__main__':
+    import json
+
+    from spiders import run_spider
+    from commons.models.data_zhejiang import DataZhejiang
+    from core.factory import ClientApp
+
+    # 爬取
+    file_path = run_spider(DataZhejiangSpider)
+    # 入库
+    data = json.loads(open(file_path, 'r', encoding='utf-8').read())
+    with ClientApp().app_context():
+        for item in data:
+            print(item)
+            DataZhejiang(**item).upsert()

web/spiders/oil.py

@@ -0,0 +1,83 @@
+import json
+
+import scrapy
+from scrapy import Request
+
+
+class OilSpider(scrapy.Spider):
+    name = "oil"
+    start_urls = [
+        ('成品油价格调整', "https://fgw.fujian.gov.cn/was5/web/search?channelid=217025&templet=advsch.jsp&sortfield=-docreltime&classsql=%25%E6%88%90%E5%93%81%E6%B2%B9%E4%BB%B7%E6%A0%BC%E8%B0%83%E6%95%B4%25*siteid%3D31*siteid%3D31&prepage=100&page=1"),
+    ]
+
+    def start_requests(self):
+        for source, url in self.start_urls:
+            yield Request(
+                method='GET',
+                url=url,
+                headers={
+                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
+                },
+                meta={'source': source, 'type': 'list'}
+            )
+
+    def parse(self, response, **kwargs):
+        if response.meta['type'] == 'list':
+            yield from self.parse_list(response)
+        elif response.meta['type'] == 'detail':
+            yield from self.parse_detail(response)
+
+    def parse_list(self, response):
+        ret = json.loads(response.text.replace('\n', ''))
+        if ret.get('count') and ret.get('docs'):
+            for item in ret['docs']:
+                if not item.get('title2'):
+                    continue
+                print(f"{item['title2']} {item['pubtime']}")
+                yield Request(
+                    method='GET',
+                    url=item['url'],
+                    meta={'source': f"{item['title2']} {item['pubtime']}", 'type': 'detail', 'time': item['pubtime']}
+                )
+
+    def parse_detail(self, response):
+        for item in response.xpath('//table[1]/tbody/tr'):
+            if len([i.get() for i in item.xpath('td/span/text()')]) > 0:
+                first_word = item.xpath('td/span/text()').get()
+                print()
+                if first_word.strip() == '油品' or first_word.strip() == '元/吨':
+                    continue
+                name = first_word
+                price, *_ = [i.get() for i in item.xpath('td/text()')]
+                yield {
+                    'name': name,
+                    'price': int(price),
+                    'date': response.meta['time'].split(' ')[0],
+                }
+            elif len([i.get() for i in item.xpath('td/text()')]) > 0:
+                first_word = item.xpath('td/text()').get()
+                if first_word.strip() == '油品' or first_word.strip() == '元/吨' or first_word.startswith('\xa0') or first_word.startswith('\n'):
+                    continue
+                name, price, *_ = [i.get() for i in item.xpath('td/text()')]
+                yield {
+                    'name': name,
+                    'price': price,
+                    'date': response.meta['time'].split(' ')[0],
+                }
+            else:
+                print()
+
+
+if __name__ == '__main__':
+    from spiders import run_spider
+    from commons.models.oil import Oil
+    from core.factory import ClientApp
+
+    # 爬取
+    file_path = run_spider(OilSpider)
+    # 入库
+    data = json.loads(open(file_path, 'r', encoding='utf-8').read())
+    with ClientApp().app_context():
+        for item in data:
+            print(item)
+            Oil(**item).upsert()

web/spiders/steel_plate.py

@@ -0,0 +1,89 @@
+import scrapy
+from scrapy import Request
+
+from commons.constants.mysteel import PageType
+
+
+class SteelPlateSpider(scrapy.Spider):
+    name = "steel_plate"
+    start_urls = [
+        (PageType.PLATE_LIST, "https://list1.mysteel.com/market/p-219-----010102-0-01010502-------1.html"),
+    ]
+    cookie = None
+    user_agent = None
+
+    def start_requests(self):
+        for source, url in self.start_urls:
+            yield Request(
+                method='GET',
+                url=url,
+                headers={
+                    'User-Agent': self.user_agent,
+                },
+                meta={'source': source, 'type': source}
+            )
+
+    def parse(self, response, **kwargs):
+        if response.meta['type'] == PageType.PLATE_LIST:
+            yield from self.parse_board_list(response)
+        if response.meta['type'] == PageType.PLATE_DETAIL:
+            yield from self.parse_board_detail(response)
+
+    def parse_board_list(self, response):
+        for item in response.xpath('//*[@id="articleList"]/ul/li/a'):
+            uri = item.xpath('@href').get()
+            name = item.xpath('text()').get()
+            print(uri, name)
+            if '福州市场中厚板价格行情' not in name:
+                continue
+            yield Request(
+                method='GET',
+                url=uri,
+                headers={
+                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
+                },
+                cookies=self.cookie,
+                meta={'source': name, 'type': PageType.PLATE_DETAIL}
+            )
+
+    def parse_board_detail(self, response):
+        title = response.xpath('//*[@id="content-title"]/text()').get()
+        date = title.split('日')[0].replace('年', '-').replace('月', '-')
+
+        for item in response.xpath('//*[@id="marketTable"]/tr[position()>2]'):
+            line = [cell.xpath('text()').get().strip() for cell in item.xpath('td')]
+            print(line)
+            if len(line) < 7:
+                continue
+            name, spec, material, source, price, fluctuating, *_ = line
+            yield {
+                'name': name,
+                'spec': spec,
+                'material': material,
+                'source': source,
+                'price': int(price),
+                'fluctuating': 0 if fluctuating == '-' else int(fluctuating),
+                'date': date
+            }
+
+
+if __name__ == '__main__':
+    import json
+
+    from spiders import run_spider, MysteelCookieTools
+    from commons.models.steel_plate import SteelPlate
+    from core.factory import ClientApp
+
+    # cookie 读取
+    cookie = MysteelCookieTools.get_cookies()
+    # 爬取
+    SteelPlateSpider.cookie = cookie
+    SteelPlateSpider.user_agent = MysteelCookieTools.user_agent
+    file_path = run_spider(SteelPlateSpider)
+    # 入库
+    data = json.loads(open(file_path, 'r', encoding='utf-8').read())
+    with ClientApp().app_context():
+        for item in data:
+            print(item)
+            SteelPlate(**item).upsert()
+

web/spiders/steel_rebar.py

@@ -0,0 +1,90 @@
+import scrapy
+from scrapy import Request
+
+from price.constants.mysteel import PageType
+
+
+class SteelRebarSpider(scrapy.Spider):
+    name = "steel_rebar"
+    start_urls = [
+        (PageType.REBAR_LIST, "https://jiancai.mysteel.com/market/pa228aa010101a0a01010502aaaa1.html"),
+    ]
+    cookie = None
+    user_agent = None
+
+    def start_requests(self):
+        for source, url in self.start_urls:
+            yield Request(
+                method='GET',
+                url=url,
+                headers={
+                    'User-Agent': self.user_agent,
+                },
+                meta={'source': source, 'type': source}
+            )
+
+    def parse(self, response, **kwargs):
+        if response.meta['type'] == PageType.REBAR_LIST:
+            yield from self.parse_steel_list(response)
+        elif response.meta['type'] == PageType.REBAR_DETAIL:
+            yield from self.parse_steel_detail(response)
+
+    def parse_steel_list(self, response):
+        for item in response.xpath('//*[@id="articleList"]/ul/li/a'):
+            uri = item.xpath('@href').get()
+            name = item.xpath('text()').get()
+            print(uri, name)
+            if ')福州市场建筑钢材价格行情' not in name:
+                continue
+            yield Request(
+                method='GET',
+                url=uri,
+                headers={
+                    'User-Agent': self.user_agent,
+                },
+                cookies=self.cookie,
+                meta={'source': name, 'type': PageType.REBAR_DETAIL}
+            )
+
+    @staticmethod
+    def parse_steel_detail(response):
+        # 解析日期
+        title = response.xpath('//*[@id="content-title"]/text()').get()
+        date = title.split('日')[0].replace('年', '-').replace('月', '-')
+        #
+        for item in response.xpath('//*[@id="marketTable"]/tr[position()>2]'):
+            line = [cell.xpath('text()').get().strip() for cell in item.xpath('td')]
+            print(line)
+            if len(line) < 8:
+                continue
+            name, spec, material, source, price, fluctuating, *_ = line
+            yield {
+                'name': name,
+                'spec': spec,
+                'material': material,
+                'source': source,
+                'price': int(price),
+                'fluctuating': 0 if fluctuating == '-' else int(fluctuating),
+                'date': date
+            }
+
+
+if __name__ == '__main__':
+    import json
+
+    from spiders import run_spider, MysteelCookieTools
+    from commons.models.steel_rebar import SteelRebar
+    from core.factory import ClientApp
+
+    # cookie 读取
+    cookie = MysteelCookieTools.get_cookies()
+    # 爬取
+    SteelRebarSpider.cookie = cookie
+    SteelRebarSpider.user_agent = MysteelCookieTools.user_agent
+    file_path = run_spider(SteelRebarSpider)
+    # 入库
+    data = json.loads(open(file_path, 'r', encoding='utf-8').read())
+    with ClientApp().app_context():
+        for item in data:
+            print(item)
+            SteelRebar(**item).upsert()

web/spiders/steel_section.py

@@ -0,0 +1,88 @@
+import scrapy
+from scrapy import Request
+
+from commons.constants.mysteel import PageType
+
+
+class SteelSectionSpider(scrapy.Spider):
+    name = "steel_section"
+    start_urls = [
+        (PageType.SECTION_LIST, "https://list1.mysteel.com/market/p-227-----010107-0-01010502-------1.html"),
+    ]
+    cookie = None
+    user_agent = None
+
+    def start_requests(self):
+        for source, url in self.start_urls:
+            yield Request(
+                method='GET',
+                url=url,
+                headers={
+                    'User-Agent': self.user_agent,
+                },
+                meta={'source': source, 'type': source}
+            )
+
+    def parse(self, response, **kwargs):
+        if response.meta['type'] == PageType.SECTION_LIST:
+            yield from self.parse_section_list(response)
+        if response.meta['type'] == PageType.SECTION_DETAIL:
+            yield from self.parse_section_detail(response)
+
+    def parse_section_list(self, response):
+        for item in response.xpath('//*[@id="articleList"]/ul/li/a'):
+            uri = item.xpath('@href').get()
+            name = item.xpath('text()').get()
+            print(uri, name)
+            if '福州市场工角槽钢价格行情' not in name:
+                continue
+            yield Request(
+                method='GET',
+                url=uri,
+                headers={
+                    'User-Agent': self.user_agent,
+                },
+                cookies=self.cookie,
+                meta={'source': name, 'type': PageType.SECTION_DETAIL}
+            )
+
+    def parse_section_detail(self, response):
+        title = response.xpath('//*[@id="content-title"]/text()').get()
+        date = title.split('日')[0].replace('年', '-').replace('月', '-')
+
+        for item in response.xpath('//*[@id="marketTable"]/tr[position()>2]'):
+            line = [cell.xpath('text()').get().strip() for cell in item.xpath('td')]
+            print(line)
+            if len(line) < 7:
+                continue
+            name, spec, material, source, price, fluctuating, *_ = line
+            yield {
+                'name': name,
+                'spec': spec,
+                'material': material,
+                'source': source,
+                'price': int(price),
+                'fluctuating': 0 if fluctuating == '-' else int(fluctuating),
+                'date': date
+            }
+
+
+if __name__ == '__main__':
+    import json
+
+    from spiders import run_spider, MysteelCookieTools
+    from commons.models.steel_section import SteelSection
+    from core.factory import ClientApp
+
+    # cookie 读取
+    cookie = MysteelCookieTools.get_cookies()
+    # 爬取
+    SteelSectionSpider.cookie = cookie
+    SteelSectionSpider.user_agent = MysteelCookieTools.user_agent
+    file_path = run_spider(SteelSectionSpider)
+    # 入库
+    data = json.loads(open(file_path, 'r', encoding='utf-8').read())
+    with ClientApp().app_context():
+        for item in data:
+            print(item)
+            SteelSection(**item).upsert()