在线文本去重工具：移除重复行和重复内容

精确去重（Set 去重）:
  输入: ["a", "b", "a", "c", "b"]
  Set: {"a", "b", "c"}
  输出: ["a", "b", "c"]

保留首次出现:
  输入: ["c", "a", "b", "a", "c"]
  输出: ["c", "a", "b"]  ← 保留第一次出现的顺序

保留末次出现:
  输入: ["c", "a", "b", "a", "c"]
  输出: ["b", "a", "c"]  ← 在末次出现位置保留，仍按输入相对序

// 精确去重（保持顺序）
function dedupeExact(text) {
  const lines = text.split('\n');
  const seen = new Set();
  return lines.filter(line => {
    if (seen.has(line)) return false;
    seen.add(line);
    return true;
  }).join('\n');
}

// 忽略大小写去重
function dedupeIgnoreCase(text) {
  const lines = text.split('\n');
  const seen = new Set();
  return lines.filter(line => {
    const lower = line.toLowerCase();
    if (seen.has(lower)) return false;
    seen.add(lower);
    return true;
  }).join('\n');
}

// 模糊去重（基于相似度；小批量可用，不适合百万行）
function dedupeFuzzy(text, threshold = 0.85) {
  const lines = text.split('\n');
  const unique = [];

  for (const line of lines) {
    if (!unique.some(u => similarity(u, line) >= threshold)) {
      unique.push(line);
    }
  }
  return unique.join('\n');
}

// 计算相似度（Jaccard 系数）
function similarity(a, b) {
  const setA = new Set(a.split(''));
  const setB = new Set(b.split(''));
  const intersection = new Set([...setA].filter(x => setB.has(x)));
  const union = new Set([...setA, ...setB]);
  return intersection.size / union.size;
}

from collections import OrderedDict

def dedupe_exact(text):
    """精确去重，保持顺序"""
    lines = text.split('\n')
    return '\n'.join(OrderedDict.fromkeys(lines))

def dedupe_ignore_case(text):
    """忽略大小写去重"""
    lines = text.split('\n')
    seen = set()
    result = []
    for line in lines:
        lower = line.lower()
        if lower not in seen:
            seen.add(lower)
            result.append(line)
    return '\n'.join(result)

def dedupe_fuzzy(text, threshold=0.85):
    """模糊去重"""
    from difflib import SequenceMatcher
    lines = text.split('\n')
    unique = []
    for line in lines:
        if not any(SequenceMatcher(None, u, line).ratio() >= threshold for u in unique):
            unique.append(line)
    return '\n'.join(unique)