Online Text Deduplication Tool: Remove Duplicate Lines and Content

Exact Dedup (Set-based):
  Input: ["a", "b", "a", "c", "b"]
  Set: {"a", "b", "c"}
  Output: ["a", "b", "c"]

Preserve First Occurrence:
  Input: ["c", "a", "b", "a", "c"]
  Output: ["c", "a", "b"]

// Exact dedup (preserve order)
function dedupeExact(text) {
  const lines = text.split('\n');
  const seen = new Set();
  return lines.filter(line => {
    if (seen.has(line)) return false;
    seen.add(line);
    return true;
  }).join('\n');
}

// Case-insensitive dedup
function dedupeIgnoreCase(text) {
  const lines = text.split('\n');
  const seen = new Set();
  return lines.filter(line => {
    const lower = line.toLowerCase();
    if (seen.has(lower)) return false;
    seen.add(lower);
    return true;
  }).join('\n');
}

// Fuzzy dedup (similarity-based)
function dedupeFuzzy(text, threshold = 0.85) {
  const lines = text.split('\n');
  const unique = [];
  for (const line of lines) {
    if (!unique.some(u => similarity(u, line) >= threshold)) {
      unique.push(line);
    }
  }
  return unique.join('\n');
}

// Jaccard similarity
function similarity(a, b) {
  const setA = new Set(a.split(''));
  const setB = new Set(b.split(''));
  const intersection = new Set([...setA].filter(x => setB.has(x)));
  const union = new Set([...setA, ...setB]);
  return intersection.size / union.size;
}

from collections import OrderedDict

def dedupe_exact(text):
    """Exact dedup, preserve order"""
    lines = text.split('\n')
    return '\n'.join(OrderedDict.fromkeys(lines))

def dedupe_ignore_case(text):
    """Case-insensitive dedup"""
    lines = text.split('\n')
    seen = set()
    result = []
    for line in lines:
        lower = line.lower()
        if lower not in seen:
            seen.add(lower)
            result.append(line)
    return '\n'.join(result)

def dedupe_fuzzy(text, threshold=0.85):
    """Fuzzy dedup"""
    from difflib import SequenceMatcher
    lines = text.split('\n')
    unique = []
    for line in lines:
        if not any(SequenceMatcher(None, u, line).ratio() >= threshold for u in unique):
            unique.append(line)
    return '\n'.join(unique)