feat: 新增相关采集任务类型

2024-07-08 10:27:57 +08:00
parent 8ccbffbbbf
commit a30edc2d46
8 changed files with 128 additions and 69 deletions
--- a/pom.xml
+++ b/pom.xml
@@ -141,6 +141,22 @@
            <artifactId>smiley-http-proxy-servlet</artifactId>
            <version>1.12.1</version>
        </dependency>
+        <!--jxls-->
+        <dependency>
+            <groupId>org.jxls</groupId>
+            <artifactId>jxls</artifactId>
+            <version>2.12.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.jxls</groupId>
+            <artifactId>jxls-poi</artifactId>
+            <version>2.12.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.jxls</groupId>
+            <artifactId>jxls-jexcel</artifactId>
+            <version>1.0.9</version>
+        </dependency>

    </dependencies>

--- a/src/main/java/mjkf/xinke/main/constant/MaterialTaskType.java
+++ b/src/main/java/mjkf/xinke/main/constant/MaterialTaskType.java
@@ -4,37 +4,69 @@ import java.util.List;

 public class MaterialTaskType {
    // 钢筋网
-    public static final Integer MY_STEEL_REBAR = 101;  // 钢筋
-    public static final Integer MY_STEEL_SECTION = 102;  // 型钢
-    public static final Integer MY_STEEL_STRAND = 103;  // 钢绞线
-    public static final Integer MY_STEEL_PLATE = 104;  // 中厚板
+    @Deprecated public static final Integer MY_STEEL_REBAR = 101;  // 钢筋
+    @Deprecated public static final Integer MY_STEEL_SECTION = 102;  // 型钢
+    @Deprecated public static final Integer MY_STEEL_STRAND = 103;  // 钢绞线
+    @Deprecated public static final Integer MY_STEEL_PLATE = 104;  // 中厚板
    // 百川网
-    public static final Integer BAIINFO_ASPHALT_DOMESTIC = 201;  // 国产沥青
-    public static final Integer BAIINFO_ASPHALT_IMPORTED = 202;  // 进口沥青
-    public static final Integer BAIINFO_CEMENT = 203;  // 水泥
+    @Deprecated public static final Integer BAIINFO_ASPHALT_DOMESTIC = 201;  // 国产沥青
+    @Deprecated public static final Integer BAIINFO_ASPHALT_IMPORTED = 202;  // 进口沥青
+    @Deprecated public static final Integer BAIINFO_CEMENT = 203;  // 水泥
    // 三明钢铁
    public static final Integer SANMING_STEEL = 301;
    // 材料信息
-    public static final Integer FUZHOU_TRANSPORTATION_BUREAU = 401;  // 福州交通局
-    public static final Integer FUZHOU_HIGHWAY_BUREAU = 402;  // 福州公路局
-    // 福建省交通工程材料调查表
-    public static final Integer FUJIAN_SURVEY = 501;
+    public static final Integer FUZHOU_TRANSPORTATION_BUREAU = 40101;  // todo 福州交通局
+    public static final Integer LONGYAN_TRANSPORTATION_BUREAU = 40102;  // todo 龙岩交通局
+    public static final Integer NANPING_TRANSPORTATION_BUREAU = 40103;  // todo 南平交通局
+    public static final Integer NINGDE_TRANSPORTATION_BUREAU = 40104;  // todo 宁德交通局
+    public static final Integer PINTAN_TRANSPORTATION_BUREAU = 40105;  // todo 平潭交通局
+    public static final Integer PUTIAN_TRANSPORTATION_BUREAU = 40106;  // todo 莆田交通局
+    public static final Integer QUANZHOU_TRANSPORTATION_BUREAU = 40107;  // todo 泉州交通局
+    public static final Integer SANMING_TRANSPORTATION_BUREAU = 40108;  // todo 三明交通局
+    public static final Integer XIAMEN_TRANSPORTATION_BUREAU = 40109;  // todo 厦门交通局
+    public static final Integer ZHANGZHOU_TRANSPORTATION_BUREAU = 40110;  // todo 漳州交通局
+    public static final Integer ZHANGZHOUKFQ_TRANSPORTATION_BUREAU = 40111;  // todo 漳州开发区交通局
+
+    public static final Integer FUZHOU_HIGHWAY_BUREAU = 40201;  // todo 福州公路局
+    public static final Integer LONGYAN_HIGHWAY_BUREAU = 40202;  // todo 龙岩公路局
+    public static final Integer NANPING_HIGHWAY_BUREAU = 40203;  // todo 南平公路局
+    public static final Integer NINGDE_HIGHWAY_BUREAU = 40204;  // todo 宁德公路局
+    public static final Integer PINTAN_HIGHWAY_BUREAU = 40205;  // todo 平潭公路局
+    public static final Integer PUTIAN_HIGHWAY_BUREAU = 40206;  // todo 莆田公路局
+    public static final Integer QUANZHOU_HIGHWAY_BUREAU = 40207;  // todo 泉州公路局
+    public static final Integer SANMING_HIGHWAY_BUREAU = 40208;  // todo 三明公路局
+    public static final Integer XIAMEN_HIGHWAY_BUREAU = 40209;  // todo 厦门公路局
+    public static final Integer ZHANGZHOU_HIGHWAY_BUREAU = 40210;  // todo 漳州公路局
+    public static final Integer ZHANGZHOUKFQ_HIGHWAY_BUREAU = 40211;  // todo 漳州开发区公路局
+    // 地市调查表
+    public static final Integer FUZHOU_SURVEY = 501;  // todo 福州调查表
+    public static final Integer LONGYAN_SURVEY = 502;  // todo 龙岩调查表
+    public static final Integer NANPING_SURVEY = 503;  // todo 南平调查表
+    public static final Integer NINGDE_SURVEY = 504;  // todo 宁德调查表
+    public static final Integer PINTAN_SURVEY = 505;  // todo 平潭调查表
+    public static final Integer PUTIAN_SURVEY = 506;  // todo 莆田调查表
+    public static final Integer QUANZHOU_SURVEY = 507;  // todo 泉州调查表
+    public static final Integer SANMING_SURVEY = 508;  // todo 三明调查表
+    public static final Integer XIAMEN_SURVEY = 509;  // todo 厦门调查表
+    public static final Integer ZHANGZHOU_SURVEY = 510;  // todo 漳州调查表
+    public static final Integer ZHANGZHOUKFQ_SURVEY = 511;  // todo 漳州开发区调查表
    // 其他省份
    public static final Integer OTHER_ZHEJIANG = 601;  // 浙江
    public static final Integer OTHER_GIANLZHOU = 602;  // 广州
    public static final Integer OTHER_YUNNAN = 603;  // 云南
+    public static final Integer OTHER_JIANGXI = 604;  // todo 江西
    // 住建厅
    public static final Integer FUJIAN_DEPARTMENT = 701;
    // 发改委
-    public static final Integer OIL = 801;  // 汽柴油
+    @Deprecated public static final Integer OIL = 801;  // 汽柴油
    // 地材（废弃）
-    public static final Integer LOCAL_MEDIUM_COARSE_SAND = 901;  // 中粗砂
-    public static final Integer LOCAL_MECHANISM_SAND = 902;  // 机制砂
-    public static final Integer LOCAL_GRAVEL = 903;  // 碎石
-    public static final Integer LOCAL_MAOTIAO_STONE = 904;  // 毛条石
-    public static final Integer LOCAL_FINE_AGGREGATE = 905;  // 细料石
-    public static final Integer LOCAL_COARSE_AGGREGATE = 906;  // 粗料石
-    public static final Integer LOCAL_CLAY = 907;  // 粘土
+    @Deprecated public static final Integer LOCAL_MEDIUM_COARSE_SAND = 901;  // 中粗砂
+    @Deprecated public static final Integer LOCAL_MECHANISM_SAND = 902;  // 机制砂
+    @Deprecated public static final Integer LOCAL_GRAVEL = 903;  // 碎石
+    @Deprecated public static final Integer LOCAL_MAOTIAO_STONE = 904;  // 毛条石
+    @Deprecated public static final Integer LOCAL_FINE_AGGREGATE = 905;  // 细料石
+    @Deprecated public static final Integer LOCAL_COARSE_AGGREGATE = 906;  // 粗料石
+    @Deprecated public static final Integer LOCAL_CLAY = 907;  // 粘土
    // 辅材
    public static final Integer ASPHALT_MODIFIER = 1001;  // 改性剂

@@ -48,23 +80,56 @@ public class MaterialTaskType {
    public static final Integer LOCAL_SANMING = 1107;  // 三明
    public static final Integer LOCAL_ZHANGZHOU = 1108;  // 漳州

+    // 网络价格
+    public static final Integer NETWORK = 1201;  // todo 网络价格
+
    public static final List<Integer> list = List.of(
-            MY_STEEL_REBAR,
-            MY_STEEL_SECTION,
-            MY_STEEL_STRAND,
-            MY_STEEL_PLATE,
-            BAIINFO_ASPHALT_DOMESTIC,
-            BAIINFO_ASPHALT_IMPORTED,
-            BAIINFO_CEMENT,
+            NETWORK,
+
            SANMING_STEEL,
+
            FUZHOU_TRANSPORTATION_BUREAU,
+            LONGYAN_TRANSPORTATION_BUREAU,
+            NANPING_TRANSPORTATION_BUREAU,
+            NINGDE_TRANSPORTATION_BUREAU,
+            PINTAN_TRANSPORTATION_BUREAU,
+            PUTIAN_TRANSPORTATION_BUREAU,
+            QUANZHOU_TRANSPORTATION_BUREAU,
+            SANMING_TRANSPORTATION_BUREAU,
+            XIAMEN_TRANSPORTATION_BUREAU,
+            ZHANGZHOU_TRANSPORTATION_BUREAU,
+            ZHANGZHOUKFQ_TRANSPORTATION_BUREAU,
+
            FUZHOU_HIGHWAY_BUREAU,
-            FUJIAN_SURVEY,
+            LONGYAN_HIGHWAY_BUREAU,
+            NANPING_HIGHWAY_BUREAU,
+            NINGDE_HIGHWAY_BUREAU,
+            PINTAN_HIGHWAY_BUREAU,
+            PUTIAN_HIGHWAY_BUREAU,
+            QUANZHOU_HIGHWAY_BUREAU,
+            SANMING_HIGHWAY_BUREAU,
+            XIAMEN_HIGHWAY_BUREAU,
+            ZHANGZHOU_HIGHWAY_BUREAU,
+            ZHANGZHOUKFQ_HIGHWAY_BUREAU,
+
+            FUZHOU_SURVEY,
+            LONGYAN_SURVEY,
+            NANPING_SURVEY,
+            NINGDE_SURVEY,
+            PINTAN_SURVEY,
+            PUTIAN_SURVEY,
+            QUANZHOU_SURVEY,
+            SANMING_SURVEY,
+            XIAMEN_SURVEY,
+            ZHANGZHOU_SURVEY,
+            ZHANGZHOUKFQ_SURVEY,
+
            OTHER_ZHEJIANG,
            OTHER_GIANLZHOU,
            OTHER_YUNNAN,
+            OTHER_JIANGXI,
+
            FUJIAN_DEPARTMENT,
-            OIL,

            LOCAL_FUZHOU,
            LOCAL_LONGYAN,
@@ -76,17 +141,25 @@ public class MaterialTaskType {
            LOCAL_ZHANGZHOU
    );

+    /**
+     * 是否来源爬虫
+     */
    public static boolean isFromSpider(Integer type) {
        var dataFromSpiderTypes = List.of(
+                NETWORK,
                OIL,
                OTHER_ZHEJIANG,
                OTHER_GIANLZHOU,
                OTHER_YUNNAN,
+                OTHER_JIANGXI,
                FUJIAN_DEPARTMENT
        );
        return dataFromSpiderTypes.contains(type);
    }

+    /**
+     * 是否地材
+     */
    public static boolean isLocalMaterial(Integer type) {
        var dataFromSpiderTypes = List.of(
                LOCAL_FUZHOU,
--- a/src/main/java/mjkf/xinke/main/service/MaterialResultService.java
+++ b/src/main/java/mjkf/xinke/main/service/MaterialResultService.java
@@ -156,7 +156,7 @@ public class MaterialResultService {
            service = fuzhouTransportationBureauService;
        } else if (type.equals(MaterialTaskType.FUZHOU_HIGHWAY_BUREAU)) {
            service = fuzhouHighwayBureauService;
-        } else if (type.equals(MaterialTaskType.FUJIAN_SURVEY)) {
+        } else if (type.equals(MaterialTaskType.FUZHOU_SURVEY)) {
            service = fujianSurveyService;
        } else if (type.equals(MaterialTaskType.OTHER_ZHEJIANG)) {
            service = dataZhejiangService;
--- a/src/main/java/mjkf/xinke/main/service/MaterialTaskService.java
+++ b/src/main/java/mjkf/xinke/main/service/MaterialTaskService.java
@@ -106,14 +106,14 @@ public class MaterialTaskService extends ServiceImpl<BaseMapper<MaterialTask>, M
            list = fuzhouTransportationBureauService.saveOrUpdateByIndexBatch(rows);
        } else if (data.getType().equals(MaterialTaskType.FUZHOU_HIGHWAY_BUREAU)) {
            list = fuzhouHighwayBureauService.saveOrUpdateByIndexBatch(rows);
-        } else if (data.getType().equals(MaterialTaskType.FUJIAN_SURVEY)) {
+        } else if (data.getType().equals(MaterialTaskType.FUZHOU_SURVEY)) {
            list = fujianSurveyService.saveOrUpdateByIndexBatch(rows);
        } else if (MaterialTaskType.isLocalMaterial(data.getType())) {
            list = localMaterialService.saveOrUpdateByIndexBatch(rows);
        } else if (data.getType().equals(MaterialTaskType.ASPHALT_MODIFIER)) {
            list = asphaltModifierService.saveOrUpdateByIndexBatch(rows);
        } else {
-            // todo 异常处理：未识别的类型
+            // todo-3 异常处理：未识别的类型
            return false;
        }
        // 保存数据副本
@@ -123,7 +123,7 @@ public class MaterialTaskService extends ServiceImpl<BaseMapper<MaterialTask>, M
    }

    private void runSpider(MaterialTask data) {
-        // todo
+        // todo-3 启动爬虫任务
    }

    private List<Row> getTableRows(File file) throws IOException {
--- a/src/main/resources/_sql/init/mysql/v0.1.sql
+++ b/src/main/resources/_sql/init/mysql/v0.1.sql
@@ -0,0 +1,3 @@
+ALTER TABLE price_publish ADD PRICE_ZHANGZHOUKFQ decimal(16,4) NULL COMMENT '漳州开发区价格';
+ALTER TABLE price_publish
+  MODIFY COLUMN PRICE_ZHANGZHOUKFQ decimal(16,4) COMMENT '漳州开发区价格' AFTER PRICE_PINTAN;
--- a/src/test/java/mjkf/xinke/MainTest.java
+++ b/src/test/java/mjkf/xinke/MainTest.java
@@ -1,23 +0,0 @@
-//
-//package mjkf.xinke;
-//
-//import org.junit.Test;
-//import org.junit.runner.RunWith;
-//import org.springframework.boot.test.context.SpringBootTest;
-//import org.springframework.test.context.junit4.SpringRunner;
-//
-///**
-// * 主测试类
-// *
-// *
-// * @date 2022/9/17 17:09
-// */
-//@RunWith(SpringRunner.class)
-//@SpringBootTest(classes = Application.class)
-//public class MainTest {
-//
-//    @Test
-//    public void test() {
-//
-//    }
-//}
--- a/src/test/java/mjkf/xinke/Test.java
+++ b/src/test/java/mjkf/xinke/Test.java
@@ -1,14 +0,0 @@
-
-package mjkf.xinke;
-
-/**
- * TestMain方法
- *
- * 
- * @date 2022/9/17 17:10
- */
-public class Test {
-    public static void main(String[] args) {
-
-    }
-}
--- a/src/test/java/mjkf/xinke/UploadNetworkPriceTest.java
+++ b/src/test/java/mjkf/xinke/UploadNetworkPriceTest.java
@@ -0,0 +1,4 @@
+package mjkf.xinke;
+
+public class UploadNetworkPriceTest {
+}