feat: 支持流式传输，改进数据集

data/src/gan.ts

@@ -8,7 +8,7 @@ const [refer] = process.argv.slice(2);
 
 let id = 0;
 
-function readMap(count: number, buffer: Buffer, h: number, w: number) {
+function readMap(count: number, arr: number[], h: number, w: number) {
     const area = w * h;
 
     const maps: number[][][] = Array.from<number[][]>({
@@ -19,7 +19,7 @@ function readMap(count: number, buffer: Buffer, h: number, w: number) {
         });
     });
 
-    buffer.subarray(4).forEach((v, i) => {
+    arr.forEach((v, i) => {
         const n = Math.floor(i / area);
         const y = Math.floor((i % area) / w);
         const x = i % w;
@@ -35,41 +35,80 @@ function generateGANData(
     map: number[][]
 ) {
     const id2 = `$${id++}`;
-    const toTrain = chooseFrom(keys, 4);
+    const toTrain = chooseFrom(keys, 30);
     const data = toTrain.map<MinamoTrainData[]>(v => {
         const floor = refer.get(v);
         if (!floor) return [];
         const size1: [number, number] = [floor.map[0].length, floor.map.length];
         const size2: [number, number] = [map[0].length, map.length];
-        if (size1[0] !== size2[0] || size1[1] !== size2[1]) return [];
+        if (size1[0] !== size2[0] || size1[1] !== size2[1]) return []; 
 
         return generateTrainData(v, id2, floor.map, map, size1);
     });
     return data.flat();
 }
 
+const enum ReceiverStatus {
+    Header,
+    Content
+}
+
+class DataReceiver {
+    static active?: DataReceiver
+    /** 接收状态 */
+    private status: ReceiverStatus = ReceiverStatus.Header;
+
+    private received: number[] = []
+    private count: number = 0;
+    private h: number = 0;
+    private w: number = 0;
+
+    receive(buf: Buffer): [number[][][], number, number, number] | null {
+        // 数据通讯 node 输入协议，单位字节：
+        // 2 - Tensor count; 1 - Map height; 1 - Map Width; N*1*H*W - Map tensor, int8 type.
+        switch (this.status) {
+            case ReceiverStatus.Header:
+                this.count = buf.readInt16BE();
+                this.h = buf.readInt8(2);
+                this.w = buf.readInt8(3);
+                this.received.push(...buf.subarray(4));
+                this.status = ReceiverStatus.Content;
+                break;
+            case ReceiverStatus.Content:
+                this.received.push(...buf);
+                break
+        }
+        if (this.received.length === this.count * this.h * this.w) {
+            delete DataReceiver.active;
+            return [readMap(this.count, this.received, this.h, this.w), this.count, this.h, this.w];
+        } else {
+            return null;
+        }
+    }
+
+    static check(buf: Buffer) {
+        if (this.active) {
+            return this.active.receive(buf);
+        } else {
+            this.active = new DataReceiver();
+            return this.active.receive(buf);
+        }
+    }
+}
+
 (async () => {
     const referTower = await readOne(refer);
     const keys = [...referTower.keys()];
 
     const client = createConnection(SOCKET_FILE, () => {
         console.log(`UDS IPC connected successfully.`);
-        // 发送四字节数据表示连接成功
-        client.write(new Uint8Array([0x00, 0x00, 0x00, 0x00]));
     });
 
     client.on('data', buffer => {
-        // 暂时不考虑流式传输，如果后续数据量非常大，再考虑优化
-        // 数据通讯 node 输入协议，单位字节：
-        // 2 - Tensor count; 1 - Map height; 1 - Map Width; N*1*H*W - Map tensor, int8 type.
-        const count = buffer.readInt16BE();
-        if (buffer.length - 4 !== count * 32 * 32) {
-            client.write(`ERROR: byte length not match.`);
-            return [];
-        }
-        const h = buffer.readInt8(2);
-        const w = buffer.readInt8(3);
-        const map = readMap(count, buffer, h, w);
+        const data = DataReceiver.check(buffer);
+        if (!data) return;
+
+        const [map, count, h, w] = data;
         const simData = map.map(v => generateGANData(keys, referTower, v));
         const rc = 0;
         const compareData = simData.flat();
@@ -83,12 +122,23 @@ function generateGANData(
         const toSend = Buffer.alloc(
             2 + // Tensor count
                 2 + // Review count
-                count + // Compare count
-                2 * (count + rc) + // Similarity data
+                1 * count + // Compare count
+                2 * 4 * (compareData.length + rc) + // Similarity data
                 compareData.length * 1 * h * w + // Compare map
                 rc * 2 * h * w, // Review map
             0
         );
+        console.log(
+            2,
+            2,
+            count,
+            2 * 4 * (compareData.length + rc),
+            compareData.length * 1 * h * w,
+            rc * 2 * h * w,
+            compareData.length,
+            rc
+        );
+        
         let offset = 0;
         toSend.writeInt16BE(count); // Tensor count
         toSend.writeInt16BE(0, 2); // Review count
@@ -98,9 +148,11 @@ function generateGANData(
             simData.map(v => v.length),
             offset
         );
-        offset += count;
+        offset += 1 * count;
         // Similarity data
         compareData.forEach(v => {
+            // console.log(v.visionSimilarity, v.topoSimilarity);
+            
             toSend.writeFloatBE(v.visionSimilarity, offset);
             offset += 4;
             toSend.writeFloatBE(v.topoSimilarity, offset);
@@ -108,15 +160,17 @@ function generateGANData(
         });
         // Compare map
         toSend.set(
-            compareData.map(v => v.map1).flat(2),
+            new Uint8Array(compareData.map(v => v.map1).flat(3)),
             offset // Set from Compare map
         );
         offset += compareData.length * 1 * h * w;
-        // Review map
-        toSend.set(
-            reviewData.map(v => [v.map1, v.map2]).flat(3),
-            offset // Set from last chunk
-        );
+        if (reviewData.length > 0) {
+            // Review map
+            toSend.set(
+                new Uint8Array(reviewData.map(v => [v.map1, v.map2]).flat(4)),
+                offset // Set from last chunk
+            );
+        }
 
         client.write(toSend);
     });

data/src/process/minamo.ts

@@ -136,7 +136,7 @@ function generateSimilarData(id: string, map: number[][]) {
     // 生成最多两个微调地图
     const width = map[0].length;
     const height = map.length;
-    const num = Math.floor(Math.random() * 1);
+    const num = Math.floor(Math.random() * 2);
     const res: [id: string, data: MinamoTrainData][] = [];
 
     for (let i = 0; i < num; i++) {