import dayjs from 'dayjs'; import customParseFormat from 'dayjs/plugin/customParseFormat'; dayjs.extend(customParseFormat); const SECOND_TIMESTAMP_REGEX = /^\d{10}(?:\.\d+)?$/; const MILLISECOND_TIMESTAMP_REGEX = /^\d{13}$/; const DATE_TIME_FORMATS = [ 'YYYYMMDDHHmmss', 'YYYYMMDDHHmm', 'YYYY-MM-DD HH:mm:ss', 'YYYY-MM-DD HH:mm', 'YYYY/MM/DD HH:mm:ss', 'YYYY/MM/DD HH:mm', 'YYYY.MM.DD HH:mm:ss', 'YYYY.MM.DD HH:mm', ]; const DATE_ONLY_FORMATS = ['YYYYMMDD', 'YYYY-MM-DD', 'YYYY/MM/DD', 'YYYY.MM.DD']; const TIME_ONLY_FORMATS = ['HHmmss', 'HH:mm:ss', 'HH.mm.ss', 'HH,mm,ss']; const PARTIAL_DATE_TIME_FORMATS = [ 'YYYY-MM-DD HH:mm:ss', 'YYYY-MM-DD HH:mm', 'YYYY/MM/DD HH:mm:ss', 'YYYY/MM/DD HH:mm', 'YYYY.MM.DD HH:mm:ss', 'YYYY.MM.DD HH:mm', 'YYYY年MM月DD日 HH:mm:ss', 'YYYY年MM月DD日 HH:mm', ]; const PARTIAL_DATE_ONLY_FORMATS = [ 'YYYY-MM-DD', 'YYYY/MM/DD', 'YYYY.MM.DD', 'YYYY年MM月DD日', ]; export type CleaningRuleKey = | 'ignoreLine' | 'dateConfig' | 'extractData' | 'extractSegment' | 'timestampConversion' | 'removeSpecialCharacters' | 'absoluteValue' | 'decimalConversion' | 'replaceValue' | 'groupingColumns' | 'magnificationAdjustment' | 'yearComplete' | 'usePreviousRowData'; export interface CleaningRuleLike { key: CleaningRuleKey; enabled?: boolean; rules?: Record; } export type PreviewRow = Record; const tryParseByFormats = (value: string, formats: string[]) => formats.map((format) => dayjs(value, format, true)).find((item) => item.isValid()) ?? null; const parseTimestampToDayjs = (value: string) => { if (SECOND_TIMESTAMP_REGEX.test(value)) { return dayjs(Number(value) * 1000); } if (MILLISECOND_TIMESTAMP_REGEX.test(value)) { return dayjs(Number(value)); } return null; }; const isPreviewRowSkipped = (rowIndex: number, headerRowIndex: number | null, ignoredRows: Set) => { if (headerRowIndex === rowIndex) { return true; } return ignoredRows.has(rowIndex + 1); }; export const splitValue = (value: unknown, separator: string, index?: number) => { const text = String(value ?? ''); if (!text || !separator) { return []; } if (index && index > 0) { return text.split(separator)[index - 1]; } return text.split(separator); }; export const sliceValue = (value: unknown, start?: number, end?: number) => { const text = String(value ?? ''); if (!text) { return ''; } return text.slice(start, end); }; export const getDateValue = (timestamp: unknown, withTime: boolean) => { const raw = String(timestamp ?? '').trim(); if (!raw) { return raw; } const date = parseTimestampToDayjs(raw); if (!date || !date.isValid()) { return raw; } if (withTime) { return date.format('YYYY-MM-DD HH:mm:ss'); } return date.format('YYYY-MM-DD'); }; export const convertDateValue = (value: unknown, type: number) => { const raw = String(value ?? ''); const normalized = raw.trim(); if (!normalized) { return ''; } const time = tryParseByFormats(normalized, TIME_ONLY_FORMATS); if (time) { return type === 3 ? time.format('HH:mm:ss') : raw; } const date = tryParseByFormats(normalized, DATE_TIME_FORMATS) || tryParseByFormats(normalized, DATE_ONLY_FORMATS) || parseTimestampToDayjs(normalized); if (!date || !date.isValid()) { return raw; } const formatted = date.format('YYYY-MM-DD HH:mm:ss'); if (type === 2) { return formatted.split(' ')[0]; } if (type === 3) { return formatted.split(' ')[1]; } return formatted; }; export const replaceStrToBlankValue = ( value: unknown, removeSpace: boolean, removeNewLine: boolean, removeText?: string, ) => { let text = String(value ?? ''); if (!text) { return ''; } if (removeSpace) { text = text.replace(/\s/g, ''); } if (removeNewLine) { text = text.replace(/\r\n/g, ''); } if (removeText) { text = text.split(String(removeText)).join(''); } return text; }; export const strToAbsoluteValue = (value: unknown, flag: unknown) => { if (!flag || flag === 'false') { return value; } const text = String(value ?? ''); if (!/^-?\d+(\.\d+)?$/.test(text)) { return value; } return Math.abs(Number(text)); }; const toBinary = (value: number) => value.toString(2); const toOctal = (value: number) => value.toString(8); const toDecimal = (value: string | number) => parseInt(String(value), 10); const toHexadecimal = (value: number) => value.toString(16); export const decimalConversionFnValue = (value: unknown, targetBase: string, currentBase: string, flag: unknown) => { if (!flag || flag === 'false' || !targetBase || !currentBase) { return value; } const parsed = parseInt(String(value ?? ''), Number(currentBase)); if (Number.isNaN(parsed)) { return value; } switch (targetBase) { case '2进制': return toBinary(parsed); case '8进制': return toOctal(parsed); case '10进制': return toDecimal(parsed); case '16进制': return toHexadecimal(parsed); default: return value; } }; export const termReplaceValue = ( value: unknown, source: string, target: string, term: string | number, flag: unknown, ) => { if (!flag || flag === 'false') { return value; } const text = String(value ?? ''); const termValue = Number(term); if (termValue === 0) { return source && text.includes(source) ? text.split(source).join(target) : text; } if (termValue === 1) { return text === source ? target : text; } return text; }; export const groupColumnsFnValue = (row: PreviewRow, groupColumns: Array>) => { return (groupColumns || []).reduce((result, item) => { const header = String(item?.header ?? ''); const direction = String(item?.direction ?? ''); const count = Number(item?.count ?? 0); const headerValue = String(row?.[header] ?? ''); if (!headerValue || !count) { return result; } const segment = direction === '前' ? headerValue.substring(0, count) : headerValue.substring(Math.max(headerValue.length - count, 0)); return result + segment; }, ''); }; export const multiplicationValue = (rawData: unknown, multiplier: number) => { if (rawData === undefined || rawData === null || rawData === '') { return rawData; } const decimalLength = String(rawData).split('.')?.[1]?.length || 0; return Number(Number(Number(rawData) * multiplier).toFixed(decimalLength)); }; export const yearCompleteFnValue = (rawData: unknown) => { const raw = String(rawData ?? ''); const text = raw.trim(); if (!text) { return rawData; } const currentYear = dayjs().year(); const candidateText = `${currentYear}-${text}`; const date = tryParseByFormats(candidateText, PARTIAL_DATE_TIME_FORMATS) || tryParseByFormats(candidateText, PARTIAL_DATE_ONLY_FORMATS); if (!date) { return rawData; } const normalizedDate = date.isAfter(dayjs()) ? date.subtract(1, 'year') : date; return /:\d{1,2}$/u.test(text) || text.includes(':') ? normalizedDate.format('YYYY-MM-DD HH:mm:ss') : normalizedDate.format('YYYY-MM-DD'); }; export const usePreviousRowDataFnValue = (rawData: unknown, list: unknown[], index: number) => { let value = rawData; let rowIndex = index; while (!value && rowIndex > 1) { value = list[rowIndex - 1]; rowIndex -= 1; } return value; }; export const resolveSplitDelimiter = (delimiterType: string, delimiterValue = '') => { if (delimiterType === 'comma') { return ','; } if (delimiterType === 'space') { return ' '; } return delimiterValue; }; export const normalizeDecimalBaseLabel = (value: unknown) => { const text = String(value ?? '').trim(); if (['2', '2进制'].includes(text)) { return '2进制'; } if (['8', '8进制'].includes(text)) { return '8进制'; } if (['10', '10进制'].includes(text)) { return '10进制'; } if (['16', '16进制'].includes(text)) { return '16进制'; } return text; }; export const escapeRegExpText = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); export const buildRemoveSpecialRegex = (specialList: string[] = [], specialStr = '') => { const patterns: string[] = []; if (specialList.includes('移除空格字符')) { patterns.push('\\s'); } if (specialList.includes('移除换行字符(tab)')) { patterns.push('\\r\\n'); } if (specialList.includes('要移除的特殊字符') && specialStr) { patterns.push(escapeRegExpText(specialStr)); } if (!patterns.length) { return ''; } return patterns.length === 1 ? patterns[0] : `(?:${patterns.join('|')})`; }; interface RunPreviewPipelineParams { rows: PreviewRow[]; headers: string[]; columnRules: CleaningRuleLike[][]; ignoredRows: Set; headerRowIndex: number | null; } export const runCleaningPreviewPipeline = ({ rows, headers, columnRules, ignoredRows, headerRowIndex, }: RunPreviewPipelineParams) => { const nextRows = JSON.parse(JSON.stringify(rows)) as PreviewRow[]; headers.forEach((header, columnIndex) => { const rules = columnRules[columnIndex] || []; if (!rules.length) { return; } rules.forEach((rule) => { if (rule.enabled === false) { return; } switch (rule.key) { case 'extractSegment': { const separator = String(rule.rules?.segment ?? ''); const segmentRow = Number(rule.rules?.segmentRow ?? 1) || 1; nextRows.forEach((row, rowIndex) => { if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) { return; } row[header] = String(splitValue(row[header], separator, segmentRow) ?? ''); }); break; } case 'extractData': { const beforeModel = Number(rule.rules?.beforeModel ?? 0); const nextModel = Number(rule.rules?.nextModel ?? 0); if (beforeModel) { nextRows.forEach((row, rowIndex) => { if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) { return; } row[header] = String(sliceValue(row[header], 0, beforeModel) ?? ''); }); } if (nextModel) { nextRows.forEach((row, rowIndex) => { if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) { return; } row[header] = String(sliceValue(row[header], -nextModel) ?? ''); }); } break; } case 'timestampConversion': { nextRows.forEach((row, rowIndex) => { if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) { return; } row[header] = String(getDateValue(row[header], true) ?? ''); }); break; } case 'dateConfig': { const dateType = Number(rule.rules?.dateType ?? 1) || 1; nextRows.forEach((row, rowIndex) => { if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) { return; } row[header] = String(convertDateValue(row[header], dateType) ?? ''); }); break; } case 'removeSpecialCharacters': { const specialList = Array.isArray(rule.rules?.specialList) ? rule.rules.specialList : []; const specialStr = String(rule.rules?.specialStr ?? ''); const removeSpace = specialList.includes('移除空格字符'); const removeNewLine = specialList.includes('移除换行字符(tab)'); const removeCustom = specialList.includes('要移除的特殊字符') ? specialStr : ''; nextRows.forEach((row, rowIndex) => { if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) { return; } row[header] = String(replaceStrToBlankValue(row[header], removeSpace, removeNewLine, removeCustom) ?? ''); }); break; } case 'absoluteValue': { const absoluteFlag = rule.rules?.absoluteData; nextRows.forEach((row, rowIndex) => { if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) { return; } row[header] = String(strToAbsoluteValue(row[header], absoluteFlag) ?? ''); }); break; } case 'decimalConversion': { const decimalFlag = rule.rules?.decimalData; const targetBase = normalizeDecimalBaseLabel(rule.rules?.decimalModel); const currentBase = normalizeDecimalBaseLabel(rule.rules?.decimalModel2).replace('进制', ''); nextRows.forEach((row, rowIndex) => { if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) { return; } row[header] = String(decimalConversionFnValue(row[header], targetBase, currentBase, decimalFlag) ?? ''); }); break; } case 'replaceValue': { const replaceFlag = rule.rules?.replaceFlag; const sourceData = String(rule.rules?.sourceData ?? ''); const targetData = String(rule.rules?.targetData ?? ''); const replaceModel = rule.rules?.replaceModel ?? 0; nextRows.forEach((row, rowIndex) => { if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) { return; } row[header] = String( termReplaceValue(row[header], sourceData, targetData, replaceModel, replaceFlag) ?? '', ); }); break; } case 'groupingColumns': { const groupColumns = Array.isArray(rule.rules?.groupColumns) ? rule.rules.groupColumns.filter((item) => { const header = String(item?.header ?? '').trim(); const direction = String(item?.direction ?? '').trim(); const count = Number(item?.count ?? 0); return !!header && !!direction && count > 0; }) : []; if (!groupColumns.length) { break; } nextRows.forEach((row, rowIndex) => { if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) { return; } row[header] = String(groupColumnsFnValue(nextRows[rowIndex], groupColumns) ?? ''); }); break; } case 'magnificationAdjustment': { const factor = Number(rule.rules?.magnificationAdjustmentValue ?? 1) || 1; nextRows.forEach((row, rowIndex) => { if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) { return; } row[header] = String(multiplicationValue(row[header], factor) ?? ''); }); break; } case 'yearComplete': { nextRows.forEach((row, rowIndex) => { if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) { return; } row[header] = String(yearCompleteFnValue(row[header]) ?? ''); }); break; } case 'usePreviousRowData': { const dataList = nextRows.map((item) => item[header]); nextRows.forEach((row, rowIndex) => { if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) { return; } row[header] = String(usePreviousRowDataFnValue(row[header], dataList, rowIndex) ?? ''); }); break; } default: break; } }); }); return nextRows; };