Sanitize a filename

The same thing in your own code

Each code block has been compiled and verified in CI. Provided under the MIT license to illustrate disarm. There is no C here: the C ABI exposes no sanitize_filename, so there is nothing to call. disarm on GitHub →

# One safe filename from a name that three platforms store differently.
#   pip install disarm
import unicodedata
from disarm import sanitize_filename

# macOS stores filenames decomposed; Windows and Linux store what they are
# given, which is usually composed. Same name on screen, different bytes.
name = "Tiếng Việt.doc"
nfc = unicodedata.normalize("NFC", name)   # Windows, Linux
nfd = unicodedata.normalize("NFD", name)   # macOS

assert nfc != nfd, "expected the two forms to differ"
assert len(nfc.encode()) == 18 and len(nfd.encode()) == 22

# Both must sanitize to one name, or a file saved on a Mac and the same file
# saved on Windows become two rows in your database.
assert sanitize_filename(nfc) == sanitize_filename(nfd) == "Tieng_Viet.doc"

# Transliteration runs before reserved-name detection, so a name that is not a
# Windows device becomes one once its accent is folded. disarm catches that.
assert sanitize_filename("áux.txt") == "_aux.txt"

print(f"ok: {len(nfc.encode())}B and {len(nfd.encode())}B converge on one name")

// One safe filename from a name that three platforms store differently.
//   cargo add disarm
use disarm::api::{normalize, sanitize_filename, NormalizationForm, Platform};

fn san(s: &str) -> String {
    sanitize_filename(s, "_", 255, Platform::Universal, None, true).unwrap()
}

fn main() {
    // macOS stores filenames decomposed; Windows and Linux store what they are
    // given, which is usually composed. Same name on screen, different bytes.
    let name = "Tiếng Việt.doc";
    let nfc = normalize(name, NormalizationForm::Nfc); // Windows, Linux
    let nfd = normalize(name, NormalizationForm::Nfd); // macOS

    assert_ne!(nfc, nfd, "expected the two forms to differ");
    assert_eq!((nfc.len(), nfd.len()), (18, 22));

    // Both must sanitize to one name, or a file saved on a Mac and the same
    // file saved on Windows become two rows in your database.
    assert_eq!(san(&nfc), "Tieng_Viet.doc");
    assert_eq!(san(&nfd), "Tieng_Viet.doc");

    // Transliteration runs before reserved-name detection, so a name that is
    // not a Windows device becomes one once its accent is folded.
    assert_eq!(san("áux.txt"), "_aux.txt");

    println!("ok: {}B and {}B converge on one name", nfc.len(), nfd.len());
}

// One safe filename from a name that three platforms store differently.
//   npm install disarm
const { sanitizeFilename } = require("disarm");

// macOS stores filenames decomposed; Windows and Linux store what they are
// given, which is usually composed. Same name on screen, different bytes.
const name = "Tiếng Việt.doc";
const nfc = name.normalize("NFC"); // Windows, Linux
const nfd = name.normalize("NFD"); // macOS

const bytes = (s) => Buffer.byteLength(s, "utf8");
if (nfc === nfd) throw new Error("expected the two forms to differ");
if (bytes(nfc) !== 18 || bytes(nfd) !== 22) throw new Error("unexpected byte lengths");

// Both must sanitize to one name, or a file saved on a Mac and the same file
// saved on Windows become two rows in your database.
if (sanitizeFilename(nfc) !== "Tieng_Viet.doc") throw new Error("NFC did not converge");
if (sanitizeFilename(nfd) !== "Tieng_Viet.doc") throw new Error("NFD did not converge");

// Transliteration runs before reserved-name detection, so a name that is not a
// Windows device becomes one once its accent is folded.
if (sanitizeFilename("áux.txt") !== "_aux.txt") throw new Error("reserved name missed");

console.log(`ok: ${bytes(nfc)}B and ${bytes(nfd)}B converge on one name`);

# One safe filename from a name that three platforms store differently.
#   gem install disarm
require "disarm"

# macOS stores filenames decomposed; Windows and Linux store what they are
# given, which is usually composed. Same name on screen, different bytes.
name = "Tiếng Việt.doc"
nfc = name.unicode_normalize(:nfc) # Windows, Linux
nfd = name.unicode_normalize(:nfd) # macOS

raise "expected the two forms to differ" if nfc == nfd
raise "unexpected byte lengths" unless [nfc.bytesize, nfd.bytesize] == [18, 22]

# Both must sanitize to one name, or a file saved on a Mac and the same file
# saved on Windows become two rows in your database.
raise "NFC did not converge" unless Disarm.sanitize_filename(nfc) == "Tieng_Viet.doc"
raise "NFD did not converge" unless Disarm.sanitize_filename(nfd) == "Tieng_Viet.doc"

# Transliteration runs before reserved-name detection, so a name that is not a
# Windows device becomes one once its accent is folded.
raise "reserved name missed" unless Disarm.sanitize_filename("áux.txt") == "_aux.txt"

puts "ok: #{nfc.bytesize}B and #{nfd.bytesize}B converge on one name"

// One safe filename from a name that three platforms store differently.
//   implementation("dev.disarm:disarm:0.14.1")
import dev.disarm.Disarm;
import java.nio.charset.StandardCharsets;
import java.text.Normalizer;

public class SanitizeFilename {
    static int bytes(String s) {
        return s.getBytes(StandardCharsets.UTF_8).length;
    }

    public static void main(String[] args) {
        // macOS stores filenames decomposed; Windows and Linux store what they
        // are given, which is usually composed. Same name, different bytes.
        String name = "Tiếng Việt.doc";
        String nfc = Normalizer.normalize(name, Normalizer.Form.NFC); // Windows, Linux
        String nfd = Normalizer.normalize(name, Normalizer.Form.NFD); // macOS

        // Java's `assert` is disabled unless the JVM is started with -ea, so
        // these throw instead.
        if (nfc.equals(nfd)) {
            throw new IllegalStateException("expected the two forms to differ");
        }
        if (bytes(nfc) != 18 || bytes(nfd) != 22) {
            throw new IllegalStateException("unexpected byte lengths");
        }

        // Both must sanitize to one name, or a file saved on a Mac and the same
        // file saved on Windows become two rows in your database.
        if (!Disarm.sanitizeFilename(nfc).equals("Tieng_Viet.doc")) {
            throw new IllegalStateException("NFC did not converge");
        }
        if (!Disarm.sanitizeFilename(nfd).equals("Tieng_Viet.doc")) {
            throw new IllegalStateException("NFD did not converge");
        }

        // Transliteration runs before reserved-name detection, so a name that
        // is not a Windows device becomes one once its accent is folded.
        if (!Disarm.sanitizeFilename("áux.txt").equals("_aux.txt")) {
            throw new IllegalStateException("reserved name missed");
        }

        System.out.printf("ok: %dB and %dB converge on one name%n", bytes(nfc), bytes(nfd));
    }
}

// One safe filename from a name that three platforms store differently.
//   implementation("dev.disarm:disarm-kotlin:0.14.1")
import dev.disarm.Disarm
import java.text.Normalizer

fun bytes(s: String) = s.toByteArray(Charsets.UTF_8).size

fun main() {
    // macOS stores filenames decomposed; Windows and Linux store what they are
    // given, which is usually composed. Same name, different bytes.
    val name = "Tiếng Việt.doc"
    val nfc = Normalizer.normalize(name, Normalizer.Form.NFC) // Windows, Linux
    val nfd = Normalizer.normalize(name, Normalizer.Form.NFD) // macOS

    check(nfc != nfd) { "expected the two forms to differ" }
    check(bytes(nfc) == 18 && bytes(nfd) == 22) { "unexpected byte lengths" }

    // Both must sanitize to one name, or a file saved on a Mac and the same
    // file saved on Windows become two rows in your database.
    check(Disarm.sanitizeFilename(nfc) == "Tieng_Viet.doc") { "NFC did not converge" }
    check(Disarm.sanitizeFilename(nfd) == "Tieng_Viet.doc") { "NFD did not converge" }

    // Transliteration runs before reserved-name detection, so a name that is
    // not a Windows device becomes one once its accent is folded.
    check(Disarm.sanitizeFilename("áux.txt") == "_aux.txt") { "reserved name missed" }

    println("ok: ${bytes(nfc)}B and ${bytes(nfd)}B converge on one name")
}