zsjz-ai/ai-frontend/packages/case/views/data/cleaningRuleUtils.ts

import dayjs from 'dayjs';
import customParseFormat from 'dayjs/plugin/customParseFormat';

dayjs.extend(customParseFormat);

const SECOND_TIMESTAMP_REGEX = /^\d{10}(?:\.\d+)?$/;
const MILLISECOND_TIMESTAMP_REGEX = /^\d{13}$/;
const DATE_TIME_FORMATS = [
  'YYYYMMDDHHmmss',
  'YYYYMMDDHHmm',
  'YYYY-MM-DD HH:mm:ss',
  'YYYY-MM-DD HH:mm',
  'YYYY/MM/DD HH:mm:ss',
  'YYYY/MM/DD HH:mm',
  'YYYY.MM.DD HH:mm:ss',
  'YYYY.MM.DD HH:mm',
];
const DATE_ONLY_FORMATS = ['YYYYMMDD', 'YYYY-MM-DD', 'YYYY/MM/DD', 'YYYY.MM.DD'];
const TIME_ONLY_FORMATS = ['HHmmss', 'HH:mm:ss', 'HH.mm.ss', 'HH,mm,ss'];
const PARTIAL_DATE_TIME_FORMATS = [
  'YYYY-MM-DD HH:mm:ss',
  'YYYY-MM-DD HH:mm',
  'YYYY/MM/DD HH:mm:ss',
  'YYYY/MM/DD HH:mm',
  'YYYY.MM.DD HH:mm:ss',
  'YYYY.MM.DD HH:mm',
  'YYYY年MM月DD日 HH:mm:ss',
  'YYYY年MM月DD日 HH:mm',
];
const PARTIAL_DATE_ONLY_FORMATS = [
  'YYYY-MM-DD',
  'YYYY/MM/DD',
  'YYYY.MM.DD',
  'YYYY年MM月DD日',
];

export type CleaningRuleKey =
  | 'ignoreLine'
  | 'dateConfig'
  | 'extractData'
  | 'extractSegment'
  | 'timestampConversion'
  | 'removeSpecialCharacters'
  | 'absoluteValue'
  | 'decimalConversion'
  | 'replaceValue'
  | 'groupingColumns'
  | 'magnificationAdjustment'
  | 'yearComplete'
  | 'usePreviousRowData';

export interface CleaningRuleLike {
  key: CleaningRuleKey;
  enabled?: boolean;
  rules?: Record<string, any>;
}

export type PreviewRow = Record<string, string>;

const tryParseByFormats = (value: string, formats: string[]) =>
  formats.map((format) => dayjs(value, format, true)).find((item) => item.isValid()) ?? null;

const parseTimestampToDayjs = (value: string) => {
  if (SECOND_TIMESTAMP_REGEX.test(value)) {
    return dayjs(Number(value) * 1000);
  }
  if (MILLISECOND_TIMESTAMP_REGEX.test(value)) {
    return dayjs(Number(value));
  }
  return null;
};

const isPreviewRowSkipped = (rowIndex: number, headerRowIndex: number | null, ignoredRows: Set<number>) => {
  if (headerRowIndex === rowIndex) {
    return true;
  }
  return ignoredRows.has(rowIndex + 1);
};

export const splitValue = (value: unknown, separator: string, index?: number) => {
  const text = String(value ?? '');
  if (!text || !separator) {
    return [];
  }
  if (index && index > 0) {
    return text.split(separator)[index - 1];
  }
  return text.split(separator);
};

export const sliceValue = (value: unknown, start?: number, end?: number) => {
  const text = String(value ?? '');
  if (!text) {
    return '';
  }
  return text.slice(start, end);
};

export const getDateValue = (timestamp: unknown, withTime: boolean) => {
  const raw = String(timestamp ?? '').trim();

  if (!raw) {
    return raw;
  }
  const date = parseTimestampToDayjs(raw);
  if (!date || !date.isValid()) {
    return raw;
  }

  if (withTime) {
    return date.format('YYYY-MM-DD HH:mm:ss');
  }

  return date.format('YYYY-MM-DD');
};

export const convertDateValue = (value: unknown, type: number) => {
  const raw = String(value ?? '');
  const normalized = raw.trim();
  if (!normalized) {
    return '';
  }
  const time = tryParseByFormats(normalized, TIME_ONLY_FORMATS);
  if (time) {
    return type === 3 ? time.format('HH:mm:ss') : raw;
  }

  const date =
    tryParseByFormats(normalized, DATE_TIME_FORMATS) ||
    tryParseByFormats(normalized, DATE_ONLY_FORMATS) ||
    parseTimestampToDayjs(normalized);

  if (!date || !date.isValid()) {
    return raw;
  }

  const formatted = date.format('YYYY-MM-DD HH:mm:ss');
  if (type === 2) {
    return formatted.split(' ')[0];
  }
  if (type === 3) {
    return formatted.split(' ')[1];
  }
  return formatted;
};

export const replaceStrToBlankValue = (
  value: unknown,
  removeSpace: boolean,
  removeNewLine: boolean,
  removeText?: string,
) => {
  let text = String(value ?? '');
  if (!text) {
    return '';
  }
  if (removeSpace) {
    text = text.replace(/\s/g, '');
  }
  if (removeNewLine) {
    text = text.replace(/\r\n/g, '');
  }
  if (removeText) {
    text = text.split(String(removeText)).join('');
  }
  return text;
};

export const strToAbsoluteValue = (value: unknown, flag: unknown) => {
  if (!flag || flag === 'false') {
    return value;
  }

  const text = String(value ?? '');
  if (!/^-?\d+(\.\d+)?$/.test(text)) {
    return value;
  }
  return Math.abs(Number(text));
};

const toBinary = (value: number) => value.toString(2);
const toOctal = (value: number) => value.toString(8);
const toDecimal = (value: string | number) => parseInt(String(value), 10);
const toHexadecimal = (value: number) => value.toString(16);

export const decimalConversionFnValue = (value: unknown, targetBase: string, currentBase: string, flag: unknown) => {
  if (!flag || flag === 'false' || !targetBase || !currentBase) {
    return value;
  }

  const parsed = parseInt(String(value ?? ''), Number(currentBase));
  if (Number.isNaN(parsed)) {
    return value;
  }

  switch (targetBase) {
    case '2进制':
      return toBinary(parsed);
    case '8进制':
      return toOctal(parsed);
    case '10进制':
      return toDecimal(parsed);
    case '16进制':
      return toHexadecimal(parsed);
    default:
      return value;
  }
};

export const termReplaceValue = (
  value: unknown,
  source: string,
  target: string,
  term: string | number,
  flag: unknown,
) => {
  if (!flag || flag === 'false') {
    return value;
  }

  const text = String(value ?? '');
  const termValue = Number(term);

  if (termValue === 0) {
    return source && text.includes(source) ? text.split(source).join(target) : text;
  }
  if (termValue === 1) {
    return text === source ? target : text;
  }
  return text;
};

export const groupColumnsFnValue = (row: PreviewRow, groupColumns: Array<Record<string, any>>) => {
  return (groupColumns || []).reduce((result, item) => {
    const header = String(item?.header ?? '');
    const direction = String(item?.direction ?? '');
    const count = Number(item?.count ?? 0);
    const headerValue = String(row?.[header] ?? '');
    if (!headerValue || !count) {
      return result;
    }

    const segment =
      direction === '前'
        ? headerValue.substring(0, count)
        : headerValue.substring(Math.max(headerValue.length - count, 0));
    return result + segment;
  }, '');
};

export const multiplicationValue = (rawData: unknown, multiplier: number) => {
  if (rawData === undefined || rawData === null || rawData === '') {
    return rawData;
  }
  const decimalLength = String(rawData).split('.')?.[1]?.length || 0;
  return Number(Number(Number(rawData) * multiplier).toFixed(decimalLength));
};

export const yearCompleteFnValue = (rawData: unknown) => {
  const raw = String(rawData ?? '');
  const text = raw.trim();
  if (!text) {
    return rawData;
  }
  const currentYear = dayjs().year();
  const candidateText = `${currentYear}-${text}`;
  const date =
    tryParseByFormats(candidateText, PARTIAL_DATE_TIME_FORMATS) ||
    tryParseByFormats(candidateText, PARTIAL_DATE_ONLY_FORMATS);
  if (!date) {
    return rawData;
  }
  const normalizedDate = date.isAfter(dayjs()) ? date.subtract(1, 'year') : date;
  return /:\d{1,2}$/u.test(text) || text.includes(':')
    ? normalizedDate.format('YYYY-MM-DD HH:mm:ss')
    : normalizedDate.format('YYYY-MM-DD');
};

export const usePreviousRowDataFnValue = (rawData: unknown, list: unknown[], index: number) => {
  let value = rawData;
  let rowIndex = index;
  while (!value && rowIndex > 1) {
    value = list[rowIndex - 1];
    rowIndex -= 1;
  }
  return value;
};

export const resolveSplitDelimiter = (delimiterType: string, delimiterValue = '') => {
  if (delimiterType === 'comma') {
    return ',';
  }
  if (delimiterType === 'space') {
    return ' ';
  }
  return delimiterValue;
};

export const normalizeDecimalBaseLabel = (value: unknown) => {
  const text = String(value ?? '').trim();
  if (['2', '2进制'].includes(text)) {
    return '2进制';
  }
  if (['8', '8进制'].includes(text)) {
    return '8进制';
  }
  if (['10', '10进制'].includes(text)) {
    return '10进制';
  }
  if (['16', '16进制'].includes(text)) {
    return '16进制';
  }
  return text;
};

export const escapeRegExpText = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');

export const buildRemoveSpecialRegex = (specialList: string[] = [], specialStr = '') => {
  const patterns: string[] = [];
  if (specialList.includes('移除空格字符')) {
    patterns.push('\\s');
  }
  if (specialList.includes('移除换行字符（tab）')) {
    patterns.push('\\r\\n');
  }
  if (specialList.includes('要移除的特殊字符') && specialStr) {
    patterns.push(escapeRegExpText(specialStr));
  }
  if (!patterns.length) {
    return '';
  }
  return patterns.length === 1 ? patterns[0] : `(?:${patterns.join('|')})`;
};

interface RunPreviewPipelineParams {
  rows: PreviewRow[];
  headers: string[];
  columnRules: CleaningRuleLike[][];
  ignoredRows: Set<number>;
  headerRowIndex: number | null;
}

export const runCleaningPreviewPipeline = ({
  rows,
  headers,
  columnRules,
  ignoredRows,
  headerRowIndex,
}: RunPreviewPipelineParams) => {
  const nextRows = JSON.parse(JSON.stringify(rows)) as PreviewRow[];

  headers.forEach((header, columnIndex) => {
    const rules = columnRules[columnIndex] || [];
    if (!rules.length) {
      return;
    }

    rules.forEach((rule) => {
      if (rule.enabled === false) {
        return;
      }
      switch (rule.key) {
        case 'extractSegment': {
          const separator = String(rule.rules?.segment ?? '');
          const segmentRow = Number(rule.rules?.segmentRow ?? 1) || 1;
          nextRows.forEach((row, rowIndex) => {
            if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) {
              return;
            }
            row[header] = String(splitValue(row[header], separator, segmentRow) ?? '');
          });
          break;
        }
        case 'extractData': {
          const beforeModel = Number(rule.rules?.beforeModel ?? 0);
          const nextModel = Number(rule.rules?.nextModel ?? 0);
          if (beforeModel) {
            nextRows.forEach((row, rowIndex) => {
              if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) {
                return;
              }
              row[header] = String(sliceValue(row[header], 0, beforeModel) ?? '');
            });
          }
          if (nextModel) {
            nextRows.forEach((row, rowIndex) => {
              if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) {
                return;
              }
              row[header] = String(sliceValue(row[header], -nextModel) ?? '');
            });
          }
          break;
        }
        case 'timestampConversion': {
          nextRows.forEach((row, rowIndex) => {
            if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) {
              return;
            }
            row[header] = String(getDateValue(row[header], true) ?? '');
          });
          break;
        }
        case 'dateConfig': {
          const dateType = Number(rule.rules?.dateType ?? 1) || 1;
          nextRows.forEach((row, rowIndex) => {
            if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) {
              return;
            }
            row[header] = String(convertDateValue(row[header], dateType) ?? '');
          });
          break;
        }
        case 'removeSpecialCharacters': {
          const specialList = Array.isArray(rule.rules?.specialList) ? rule.rules.specialList : [];
          const specialStr = String(rule.rules?.specialStr ?? '');
          const removeSpace = specialList.includes('移除空格字符');
          const removeNewLine = specialList.includes('移除换行字符（tab）');
          const removeCustom = specialList.includes('要移除的特殊字符') ? specialStr : '';

          nextRows.forEach((row, rowIndex) => {
            if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) {
              return;
            }
            row[header] = String(replaceStrToBlankValue(row[header], removeSpace, removeNewLine, removeCustom) ?? '');
          });
          break;
        }
        case 'absoluteValue': {
          const absoluteFlag = rule.rules?.absoluteData;
          nextRows.forEach((row, rowIndex) => {
            if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) {
              return;
            }
            row[header] = String(strToAbsoluteValue(row[header], absoluteFlag) ?? '');
          });
          break;
        }
        case 'decimalConversion': {
          const decimalFlag = rule.rules?.decimalData;
          const targetBase = normalizeDecimalBaseLabel(rule.rules?.decimalModel);
          const currentBase = normalizeDecimalBaseLabel(rule.rules?.decimalModel2).replace('进制', '');
          nextRows.forEach((row, rowIndex) => {
            if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) {
              return;
            }
            row[header] = String(decimalConversionFnValue(row[header], targetBase, currentBase, decimalFlag) ?? '');
          });
          break;
        }
        case 'replaceValue': {
          const replaceFlag = rule.rules?.replaceFlag;
          const sourceData = String(rule.rules?.sourceData ?? '');
          const targetData = String(rule.rules?.targetData ?? '');
          const replaceModel = rule.rules?.replaceModel ?? 0;
          nextRows.forEach((row, rowIndex) => {
            if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) {
              return;
            }
            row[header] = String(
              termReplaceValue(row[header], sourceData, targetData, replaceModel, replaceFlag) ?? '',
            );
          });
          break;
        }
        case 'groupingColumns': {
          const groupColumns = Array.isArray(rule.rules?.groupColumns)
            ? rule.rules.groupColumns.filter((item) => {
                const header = String(item?.header ?? '').trim();
                const direction = String(item?.direction ?? '').trim();
                const count = Number(item?.count ?? 0);
                return !!header && !!direction && count > 0;
              })
            : [];
          if (!groupColumns.length) {
            break;
          }
          nextRows.forEach((row, rowIndex) => {
            if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) {
              return;
            }
            row[header] = String(groupColumnsFnValue(nextRows[rowIndex], groupColumns) ?? '');
          });
          break;
        }
        case 'magnificationAdjustment': {
          const factor = Number(rule.rules?.magnificationAdjustmentValue ?? 1) || 1;
          nextRows.forEach((row, rowIndex) => {
            if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) {
              return;
            }
            row[header] = String(multiplicationValue(row[header], factor) ?? '');
          });
          break;
        }
        case 'yearComplete': {
          nextRows.forEach((row, rowIndex) => {
            if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) {
              return;
            }
            row[header] = String(yearCompleteFnValue(row[header]) ?? '');
          });
          break;
        }
        case 'usePreviousRowData': {
          const dataList = nextRows.map((item) => item[header]);
          nextRows.forEach((row, rowIndex) => {
            if (isPreviewRowSkipped(rowIndex, headerRowIndex, ignoredRows)) {
              return;
            }
            row[header] = String(usePreviousRowDataFnValue(row[header], dataList, rowIndex) ?? '');
          });
          break;
        }
        default:
          break;
      }
    });
  });

  return nextRows;
};