Remove invisible characters from text

The same thing in your own code

Each code block has been compiled and verified in CI. Provided under the MIT license to illustrate disarm. disarm on GitHub →

# Remove every class of invisible character disarm exposes in all bindings.
#   pip install disarm
from disarm import (
    strip_zero_width_chars, strip_bidi, strip_tags,
    strip_variation_selectors, strip_noncharacters,
    strip_pua, strip_control_chars,
)

# The characters hidden in the sample below. Printing stripped text proves
# nothing — it looks the same either way — so assert they are gone instead.
HOSTILE = "​‌⁠‮‬️\U000e0041"

text = "pay​pal‌ ad⁠min ‮reversed‬ text️\U000e0041"

for step in (strip_zero_width_chars, strip_bidi, strip_tags,
             strip_variation_selectors, strip_noncharacters,
             strip_pua, strip_control_chars):
    text = step(text)

for ch in HOSTILE:
    assert ch not in text, f"U+{ord(ch):04X} survived"

print(f'ok: {len(HOSTILE)} hostile codepoints absent from "{text}"')

// Remove every class of invisible character disarm exposes in all bindings.
//   cargo add disarm
use disarm::api::*;

fn main() {
    // The characters hidden in the sample below. Printing stripped text proves
    // nothing — it looks the same either way — so assert they are gone instead.
    const HOSTILE: [char; 8] = [
        '\u{200b}', '\u{200c}', '\u{feff}', '\u{2060}',
        '\u{202e}', '\u{202c}', '\u{fe0f}', '\u{e0041}',
    ];

    let text = "pay\u{200b}pal\u{200c}\u{feff} ad\u{2060}min \u{202e}reversed\u{202c} text\u{fe0f}\u{e0041}";

    let text = strip_zero_width_chars(text);
    let text = strip_bidi(&text);
    let text = strip_tags(&text);
    let text = strip_variation_selectors(&text);
    let text = strip_noncharacters(&text);
    let text = strip_pua(&text);
    let text = strip_control_chars(&text);

    for ch in HOSTILE {
        assert!(!text.contains(ch), "U+{:04X} survived", ch as u32);
    }

    println!(r#"ok: {} hostile codepoints absent from "{}""#, HOSTILE.len(), text);
}

// Remove every class of invisible character disarm exposes in all bindings.
//   npm i disarm
const assert = require("node:assert/strict");
const {
  stripZeroWidthChars, stripBidi, stripTags,
  stripVariationSelectors, stripNoncharacters,
  stripPua, stripControlChars,
} = require("disarm");

// The characters hidden in the sample below. Printing stripped text proves
// nothing — it looks the same either way — so assert they are gone instead.
const HOSTILE = [..."​‌⁠‮‬️\u{E0041}"];

let text = "pay​pal‌ ad⁠min ‮reversed‬ text️\u{E0041}";

for (const step of [stripZeroWidthChars, stripBidi, stripTags,
                    stripVariationSelectors, stripNoncharacters,
                    stripPua, stripControlChars]) {
  text = step(text);
}

for (const ch of HOSTILE) {
  assert.ok(!text.includes(ch), `U+${ch.codePointAt(0).toString(16).toUpperCase()} survived`);
}

console.log(`ok: ${HOSTILE.length} hostile codepoints absent from "${text}"`);

# Remove every class of invisible character disarm exposes in all bindings.
#   gem install disarm
require "disarm"

# The characters hidden in the sample below. Printing stripped text proves
# nothing — it looks the same either way — so assert they are gone instead.
HOSTILE = "​‌⁠‮‬️\u{E0041}".chars

text = "pay​pal‌ ad⁠min ‮reversed‬ text️\u{E0041}"

%i[strip_zero_width_chars strip_bidi strip_tags
   strip_variation_selectors strip_noncharacters
   strip_pua strip_control_chars].each do |step|
  text = Disarm.public_send(step, text)
end

HOSTILE.each do |ch|
  raise format("U+%04X survived", ch.ord) if text.include?(ch)
end

puts %(ok: #{HOSTILE.length} hostile codepoints absent from "#{text}")

// Remove every class of invisible character disarm exposes in all bindings.
//   implementation("dev.disarm:disarm:0.14.1")
import dev.disarm.Disarm;

public class StripInvisibles {
    public static void main(String[] args) {
        // The characters hidden in the sample below. Printing stripped text
        // proves nothing — it looks the same either way — so check they are
        // gone instead. Java's `assert` is disabled unless the JVM is started
        // with -ea, so this throws rather than asserting.
        int[] hostile = {
            0x200B, 0x200C, 0xFEFF, 0x2060,
            0x202E, 0x202C, 0xFE0F, 0xE0041,
        };

        // U+E0041 is above the BMP, so Java spells it as a surrogate pair.
        String text = "pay​pal‌ ad⁠min ‮reversed‬ text️󠁁";

        text = Disarm.stripZeroWidthChars(text);
        text = Disarm.stripBidi(text);
        text = Disarm.stripTags(text);
        text = Disarm.stripVariationSelectors(text);
        text = Disarm.stripNoncharacters(text);
        text = Disarm.stripPua(text);
        text = Disarm.stripControlChars(text);

        for (int cp : hostile) {
            if (text.codePoints().anyMatch(c -> c == cp)) {
                throw new AssertionError(String.format("U+%04X survived", cp));
            }
        }

        System.out.printf("ok: %d hostile codepoints absent from \"%s\"%n",
                          hostile.length, text);
    }
}

// Remove every class of invisible character disarm exposes in all bindings.
//   implementation("dev.disarm:disarm-kotlin:0.14.1")
import dev.disarm.kotlin.*

fun main() {
    // The characters hidden in the sample below. Printing stripped text proves
    // nothing — it looks the same either way — so check they are gone instead.
    val hostile = listOf(
        0x200B, 0x200C, 0xFEFF, 0x2060,
        0x202E, 0x202C, 0xFE0F, 0xE0041,
    )

    // U+E0041 is above the BMP, so it is spelled as a surrogate pair.
    val text = "pay​pal‌ ad⁠min ‮reversed‬ text️󠁁"
        .stripZeroWidthChars()
        .stripBidi()
        .stripTags()
        .stripVariationSelectors()
        .stripNoncharacters()
        .stripPua()
        .stripControlChars()

    val survivors = text.codePoints().toArray().toSet()
    hostile.forEach { cp ->
        check(cp !in survivors) { "U+%04X survived".format(cp) }
    }

    println("ok: ${hostile.size} hostile codepoints absent from \"$text\"")
}

/* Remove every class of invisible character disarm exposes in all bindings.
 *
 * The C ABI is not published to any registry, so this links against a cdylib
 * built from bindings/cabi in the disarm repository. Every disarm_* function
 * here returns an owned string the caller frees with disarm_string_free.
 */
#include <stdio.h>
#include <string.h>
#include "disarm.h"

/* The characters hidden in the sample below, as UTF-8. Printing stripped text
 * proves nothing — it looks the same either way — so search for them instead. */
static const char *HOSTILE[] = {
    "​", "‌", "", "⁠",
    "‮", "‬", "️", "\U000E0041",
};
#define HOSTILE_LEN (sizeof HOSTILE / sizeof *HOSTILE)

int main(void) {
    const char *input =
        "pay​pal‌ ad⁠min ‮reversed‬ text️\U000E0041";

    char *(*steps[])(const char *) = {
        disarm_strip_zero_width_chars, disarm_strip_bidi, disarm_strip_tags,
        disarm_strip_variation_selectors, disarm_strip_noncharacters,
        disarm_strip_pua, disarm_strip_control_chars,
    };

    char *text = disarm_strip_zero_width_chars(input);
    for (size_t i = 1; i < sizeof steps / sizeof *steps; i++) {
        char *next = steps[i](text);
        disarm_string_free(text);
        text = next;
    }

    for (size_t i = 0; i < HOSTILE_LEN; i++) {
        if (strstr(text, HOSTILE[i]) != NULL) {
            fprintf(stderr, "a hostile codepoint survived\n");
            disarm_string_free(text);
            return 1;
        }
    }

    printf("ok: %zu hostile codepoints absent from \"%s\"\n", HOSTILE_LEN, text);
    disarm_string_free(text);
    return 0;
}