保存进度！

README.md

@@ -1,3 +1,12 @@
-# east-utils
+<h1 style="font-size:4rem;text-align:center;color:red;" align="center" title >EAST 数据处理工具</h1>
 
-EAST数据处理工具。
+<p align="center"></p>
+
+# 概述
+
+这是产险厦门分公司用于处理EAST数据的工具。
+
+## 功能描述
+
+-   拆分过大的txt文件；
+-   文件转换成excel表格用于核对；

code/east-datafile-utils/.editorconfig

@@ -0,0 +1,25 @@
+# top-most EditorConfig file
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 4
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.js]
+indent_size = 4
+
+[*.ts]
+indent_size = 4
+
+[*.py]
+indent_size = 4
+
+[*.java]
+indent_size = 4
+
+[*.xml]
+indent_size = 2

code/east-datafile-utils/.vscode/fileheader.template.js

@@ -1,12 +1,3 @@
-/*
- * @Author: Kane
- * @Date: 2025-03-16 00:35:09
- * @LastEditors: Kane
- * @FilePath: /cpicxim-huixiabao/.vscode/fileheader.template.js
- * @Description: 
- * 
- * Copyright (c) ${2023} by Kane, All Rights Reserved. 
- */
 /**
  * This file is generated by VSCode extension: Fileheader Pro
  */
@@ -26,10 +17,10 @@
  * @property {string} dirPath the directory path, relative to project root with POSIX path separator
  * @property {string} fileName filename with extension
  */
- 
+
  /**
  * @typedef {string | number | null | undefined | Template | boolean} TemplateInterpolation NOTE: boolean or falsy value will render empty string
- * 
+ *
  * @typedef {{ strings: TemplateStringsArray; interpolations: TemplateInterpolation[]; }} Template
  * @typedef {(strings: TemplateStringsArray, ...values: any[]) => string} ITemplateFunction
  *
@@ -71,14 +62,14 @@ class CustomLanguageProvider extends globalThis.FileheaderLanguageProvider {
     // prettier-ignore
     return tpl
 `/**
- * @Author:        ${variables.authorName} <${variables.authorEmail}>
+ * @Author: ${variables.authorName} <${variables.authorEmail}>
- * @Date:          ${variables.birthtime}
+ * @Date: ${variables.birthtime}
- * @LastEditors:   ${variables.userName}
+ * @LastEditors: ${variables.userName}
- * @LastModified:  ${variables.mtime}
+ * @LastModified: ${variables.mtime}
- * @FilePath:      ${variables.filePath}
+ * @FilePath: ${variables.filePath}
  * @Description:
- * 
+ *
- * Copyright (c) ${2025} by Kane All rights reserved
+ *               Copyright (c) ${2025} by Kane All rights reserved
  */`;
   }
 }

code/east-datafile-utils/.vscode/launch.json

@@ -0,0 +1,30 @@
+{
+    // 使用 IntelliSense 了解相关属性。
+    // 悬停以查看现有属性的描述。
+    // 欲了解更多信息，请访问: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+
+        {
+            "type": "java",
+            "name": "Current File",
+            "request": "launch",
+            "mainClass": "${file}"
+        },
+        {
+            "type": "java",
+            "name": "App",
+            "request": "launch",
+            "mainClass": "com.cpic.xim.App",
+            "projectName": "east-utils"
+        },
+        {
+            "type": "java",
+            "name": "Debug",
+            "request": "launch",
+            "mainClass": "com.cpic.xim.App",
+            "projectName": "east-utils",
+            "args": ["-Xms512m", "-Xmx8g"]
+        }
+    ]
+}

code/east-datafile-utils/pom.xml

@@ -8,15 +8,15 @@
   <artifactId>east-utils</artifactId>
   <version>20250808.1921</version>
 
-  <name>east</name>
+  <name>east-utils</name>
-  <!-- FIXME change it to the project's website -->
+  <url>http://www.cpic.com.cn</url>
-  <url>http://www.example.com</url>
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <maven.compiler.release>17</maven.compiler.release>
-    <log4j.version>2.24.3</log4j.version>
+    <log4j.version>2.25.1</log4j.version>
     <jackson.version>2.19.2</jackson.version>
+    <!-- <jackson.version>LATEST</jackson.version> -->
   </properties>
 
   <dependencyManagement>
@@ -43,6 +43,7 @@
       <artifactId>junit-jupiter-params</artifactId>
       <scope>test</scope>
     </dependency>
+
     <!-- POI -->
     <dependency>
       <groupId>org.apache.poi</groupId>
@@ -64,7 +65,8 @@
       <artifactId>poi-ooxml-schemas</artifactId>
       <version>4.1.2</version>
     </dependency>
- <!--jackson-->
+
+    <!--jackson-->
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
@@ -82,6 +84,23 @@
       <artifactId>jackson-annotations</artifactId>
       <version>${jackson.version}</version>
     </dependency>
+
+    <!-- log-4j -->
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-core</artifactId>
+      <version>${log4j.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j2-impl</artifactId>
+      <version>${log4j.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-api</artifactId>
+      <version>${log4j.version}</version>
+    </dependency>
   </dependencies>
 
   <build>
@@ -149,4 +168,4 @@
       </plugin>
     </plugins>
   </build>
-</project>
+</project>

code/east-datafile-utils/readme.md

@@ -0,0 +1,13 @@
+<h1 style="font-size:4rem;text-align:center;color:red;" align="center" title >EAST 数据处理工具</h1>
+
+
+
+# 概述
+
+这是产险厦门分公司用于处理EAST数据的工具。
+
+## 功能描述
+
+-   拆分过大的txt文件；
+-   文件转换成excel表格用于核对；
+

code/east-datafile-utils/src/main/java/com/cpic/xim/App.java

@@ -5,16 +5,14 @@
  * @LastModified: 2025-08-08 19:05:06
  * @FilePath: src/main/java/com/cpic/xim/App.java
  * @Description:
- * 
+ *
  *               Copyright (c) 2025 by Kane All rights reserved
  */
 package com.cpic.xim;
 
 import java.io.File;
+import com.cpic.xim.east.utils.DataFileSpliter;
 
-/**
- * Hello world!
- */
 public class App
 {
     private static final int    MAX_LINE_COUNT_PER_FILE = 250000;
@@ -24,9 +22,16 @@ public class App
     {
         String fileNameBDJBXXB = "L:\\east\\MD5_BDJCXX_BDJBXXB_RESULT_3080100.txt";
 
-        com.cpic.xim.east.utils.DataFileSpliter.splitFile( new File( fileNameBDJBXXB ),
+        try
-                                                           MAX_LINE_COUNT_PER_FILE,
+        {
-                                                           TITLE_BDJBXXB );
+            DataFileSpliter.splitFile( new File( fileNameBDJBXXB ),
+                                       MAX_LINE_COUNT_PER_FILE,
+                                       TITLE_BDJBXXB );
+        }
+        catch ( Exception error )
+        {
+            error.printStackTrace();
+        }
 
         System.out.println( "拆完了！" );
     }

code/east-datafile-utils/src/main/java/com/cpic/xim/east/utils/DataFileConverter.java

@@ -0,0 +1,140 @@
+/**
+ * @Author: Kane Wang <wangkane@qq.com>
+ * @Date: 2025-08-09 01:09:26
+ * @LastEditors: Kane Wang
+ * @LastModified: 2025-08-11 15:54:24
+ * @FilePath: src/main/java/com/cpic/xim/east/utils/DataFileConverter.java
+ * @Description:
+ *
+ *               Copyright (c) 2025 by Kane All rights reserved
+ */
+package com.cpic.xim.east.utils;
+
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.usermodel.Workbook;
+import org.apache.poi.xssf.streaming.SXSSFWorkbook;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DataFileConverter
+{
+    private static Logger logger = LoggerFactory.getLogger( DataFileConverter.class );
+
+    /**
+     * 将txt格式的east数据文件，转换成excel文件。
+     * 逐行转换，每行数据用split拆分。
+     * 使用SXSSFWorkbook格式的对象。用于转换超大规模excel文件。
+     *
+     * @param eastDataFilePath east数据文件路径
+     * @param xlsxFilePath     输出的excel文件路径
+     * @param spliterString    数据单元正则表达式分隔符
+     * @param sheetName        sheet名称
+     * @throws FileNotFoundException
+     * @throws IOException
+     */
+    public static void convertEASTDataFileToXLSX(
+            String eastDataFilePath,
+            String xlsxFilePath,
+            String spliterString,
+            String sheetName
+    )
+      throws FileNotFoundException,
+      IOException
+    {
+        FileOutputStream xlsxFile = null;
+        BufferedReader   reader   = null;
+        Workbook         workbook = null;
+        Sheet            sheet    = null;
+
+        String line;
+        int    rowIndex = 0;
+
+        try
+        {
+            reader = new BufferedReader( new FileReader( eastDataFilePath ) );
+        }
+        catch ( IOException error )
+        {
+            logger.error( "打开文件" + eastDataFilePath + "失败！！！！" );
+
+            throw error;
+        }
+
+        try
+        {
+            workbook = new SXSSFWorkbook();
+            sheet    = workbook.createSheet( sheetName );
+
+            while ( (line = reader.readLine()) != null )
+            {
+                String[] rowData   = line.split( spliterString );
+                Row      row       = sheet.createRow( rowIndex );
+                int      cellIndex = 0;
+
+                for ( String cellData : rowData )
+                {
+                    row.createCell( cellIndex++ ).setCellValue( cellData );
+                }
+
+                rowIndex++;
+            }
+
+            xlsxFile = new FileOutputStream( xlsxFilePath );
+            workbook.write( xlsxFile );
+
+            xlsxFile.close();
+            workbook.close();
+            reader.close();
+        }
+        catch ( IOException error )
+        {
+            throw error;
+        }
+        finally
+        {
+            if ( xlsxFile != null )
+            {
+                try
+                {
+                    xlsxFile.close();
+                }
+                catch ( IOException error )
+                {
+                    error.printStackTrace();
+                }
+            }
+
+            if ( workbook != null )
+            {
+                try
+                {
+                    workbook.close();
+                }
+                catch ( IOException error )
+                {
+                    error.printStackTrace();
+                }
+            }
+
+            if ( reader != null )
+            {
+                try
+                {
+                    reader.close();
+                }
+                catch ( IOException error )
+                {
+                    error.printStackTrace();
+                }
+            }
+        }
+
+    }
+}

code/east-datafile-utils/src/main/java/com/cpic/xim/east/utils/DataFileSpliter.java

@@ -1,11 +1,11 @@
 /**
  * @Author: Kane Wang <wangkane@qq.com>
- * @Date: 2025-08-08 17:23:44
+ * @Date: 2025-08-09 00:10:12
  * @LastEditors: Kane Wang
- * @LastModified: 2025-08-08 17:25:07
+ * @LastModified: 2025-08-09 12:06:41
  * @FilePath: src/main/java/com/cpic/xim/east/utils/DataFileSpliter.java
- * @Description: 
+ * @Description:
- * 
+ *
  *               Copyright (c) 2025 by Kane All rights reserved
  */
 package com.cpic.xim.east.utils;
@@ -13,9 +13,12 @@ package com.cpic.xim.east.utils;
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * @author Kane
@@ -23,8 +26,14 @@ import java.io.IOException;
  */
 public class DataFileSpliter
 {
+    private static Logger logger = LoggerFactory.getLogger( DataFileSpliter.class );
 
-    public static void splitFile( File inputFile, int lineCountPerFile , String titleString )
+    public static void splitFile( File inputFile,
+                                  int lineCountPerFile,
+                                  String titleString
+    )
+      throws FileNotFoundException,
+      IOException
     {
         try (
                 BufferedReader reader = new BufferedReader( new FileReader( inputFile ) )
@@ -45,7 +54,7 @@ public class DataFileSpliter
                     fileCount++;
                     lineCount = 0;
 
-                    writer    = createWriterForFile( inputFile, fileCount );
+                    writer = createWriterForFile( inputFile, fileCount );
                 }
 
                 // 如果是第一行，则写入标题
@@ -61,9 +70,11 @@ public class DataFileSpliter
             }
             writer.close();
         }
-        catch ( IOException e )
+        catch ( IOException error )
         {
-            e.printStackTrace();
+            logger.error( "出现异常" + error.getMessage() );
+
+            throw error;
         }
     }
 
@@ -73,7 +84,7 @@ public class DataFileSpliter
         {
             String     outputFileName = String.format( "%s_part%d.txt", inputFile.getName(), fileCount );
             FileWriter fileWriter     = new FileWriter( new File( inputFile.getParent(), outputFileName ) );
-            
+
             return new BufferedWriter( fileWriter );
         }
         catch ( IOException e )

code/east-datafile-utils/src/main/resources/config.json

@@ -0,0 +1,3 @@
+{
+    "version": "1.0"
+}

code/east-datafile-utils/src/main/resources/log4j2.xml

@@ -0,0 +1,40 @@
+<Configuration status="WARN" monitorInterval="300">
+    <Appenders>
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} ### %msg%n" />
+            <ThresholdFilter level="trace" onMatch="ACCEPT" onMismatch="DENY" />
+        </Console>
+        <RollingFile name="rolling_file_win"
+            filePattern="./logs/$${date:yyyy-MM}/huixiabao-%d{MM-dd-yyyy}-%i.log.gz">
+            <PatternLayout>
+                <Pattern>[%t][%level][%d{HH:mm:ss.SSS}][%logger.%M{36}#%L] %msg%n</Pattern>
+            </PatternLayout>
+            <Policies>
+                <TimeBasedTriggeringPolicy interval="1" />
+                <SizeBasedTriggeringPolicy size="20MB" />
+                <DefaultRolloverStrategy max="20" />
+            </Policies>
+        </RollingFile>
+        <RollingFile name="rolling_file_linux"
+            filePattern="/logs/east/$${date:yyyy-MM}/app-%d{MM-dd-yyyy}-%i.log.gz">
+            <PatternLayout>
+                <Pattern>[%t][%level][%d{HH:mm:ss.SSS}][%logger.%M{36}#%L] %msg%n</Pattern>
+            </PatternLayout>
+            <Policies>
+                <TimeBasedTriggeringPolicy interval="1" />
+                <SizeBasedTriggeringPolicy size="20MB" />
+                <DefaultRolloverStrategy max="20" />
+            </Policies>
+        </RollingFile>
+    </Appenders>
+    <Loggers>
+        <!-- <Logger name="mylog" level="info">
+            <AppenderRef ref="rolling_file" />
+        </Logger> -->
+        <Root level="debug">
+            <AppenderRef ref="rolling_file_linux" />
+            <AppenderRef ref="rolling_file_win" />
+            <AppenderRef ref="Console" />
+        </Root>
+    </Loggers>
+</Configuration>

文档/EAST表结构/dbload_dm3.txt

@@ -0,0 +1,68 @@
+
+
+
+dir="/cpic/cpicapp/east"
+dmdir="/cpic/cpicapp/dm3/bin"
+cd $dir
+
+if [ $# -ne 1 ];then
+cat << EOF
+Usage: $0 dirname 
+       dirname 
+EOF
+exit 1
+fi
+dirname=$1
+
+
+list=`ls ${dirname}/*.txt`
+for line in $list
+##cat table.list|while read line
+do
+   filename=${line#*/}
+   tabname="EAST_"${filename%-*}
+   echo $tabname
+
+    ##将表名转换为小写
+    c_tabname=`echo "${tabname}"|tr [A-Z] [a-z]`
+   
+    ##c_tabname=`echo "${tabname}"`
+    echo $c_tabname
+
+
+   result=`${dmdir}/disql ****/'"********"'@*******:***** << ! 
+        set heading off
+        set feedback off
+        set pagesize 0
+        set verify off
+        set echo off
+        set lineshow off
+        SET TIMING off
+
+         select '|'||trim(LISTAGG(column_name,',')within group(order by column_id))||'|' from (select column_name||'#'||case when data_type in('CHAR','VARCHAR2','NVARCHAR2','VARCHAR','TEXT') AND DATA_LENGTH > 255 THEN 'CHAR(8000)'  ELSE '' END column_name,column_id from user_tab_columns ta  where table_name = upper('${tabname}'));
+
+       exit
+!`
+
+    echo $result>clumns01.txt
+    result2=`cut -d$'|' -f2 clumns01.txt`
+    echo $result2>clumns01.txt
+    sed s/[[:space:]]//g clumns01.txt>clumns.txt
+    sed -i 's/#/ /g' clumns.txt
+
+     ##拼装control文件
+    echo "load data">tmp.ctl
+    echo "infile '${dir}/import20250630/${filename}'">>tmp.ctl
+    ##echo "infile '${dir}/import20250228/${c_tabname}.txt'">>tmp.ctl
+    echo "badfile '${dir}/bad/${c_tabname}.bad'">>tmp.ctl
+    echo "into table ${c_tabname}">>tmp.ctl
+    echo "fields terminated by '	'" >> tmp.ctl
+    ##echo "fields  '	'">>tmp.ctl
+    echo "(">>tmp.ctl
+    cat clumns.txt>>tmp.ctl
+    echo ")">>tmp.ctl
+
+   ${dmdir}/dmfldr userid=****/'"*********"'@********:*****  control=\'${dir}/tmp.ctl\' null_mode=true
+
+done
+