path: root/bin/sz-rm-meta.sh

#!/usr/bin/env bash
set -euo pipefail

# sz-rm-meta.sh — strip metadata from image files.
#
#   sz-rm-meta.sh [options] FILE...
#
#   -l, --list        report what would be removed; change nothing
#   -o, --out DIR     write cleaned copies into DIR instead of editing in place
#       --keep-icc    keep the embedded ICC colour profile (stripped by default)
#       --keep-xattr  leave extended attributes alone (macOS: kMDItemWhereFroms,
#                     com.apple.quarantine — these record the download URL)
#   -d, --destructive re-encode the pixels as well (see "Two modes" below)
#       --quality N   JPEG quality for --destructive (default 85)
#       --scale PCT   resize percentage for --destructive (default 97)
#       --rename      rename the file to a random name, keeping the extension
#   -q, --quiet       only report failures
#
# Two modes, because "no metadata" and "not traceable" are different jobs:
#
#   default       Lossless. Rewrites the container, copies the compressed image
#                 data byte for byte. Pixels stay bit-identical, so the file is
#                 still matchable against the source by perceptual hash, and the
#                 quantization/Huffman tables still fingerprint the encoder that
#                 produced it. Right for sanitising an archive you keep.
#
#   --destructive Re-encodes: resize by --scale, re-compress at --quality, fresh
#                 quantization tables, then run the lossless pass over the
#                 result. Breaks bitstream-level linkage to the source copy.
#                 Does NOT defeat reverse-image search or perceptual hashing —
#                 nothing that preserves what the picture looks like can. Needs
#                 ImageMagick.
#
# What the lossless pass removes, per format:
#   JPEG  every APPn segment except JFIF (APP0) and Adobe (APP14), all COM
#         comments, and any trailer appended after the EOI marker. That covers
#         EXIF (camera, GPS, serial, timestamps), XMP, IPTC and the Photoshop
#         8BIM block (by-line, credit, document ancestry).
#   PNG   everything outside an allowlist of image and colour chunks. An
#         allowlist rather than a denylist because private chunks are the worst
#         offenders: Fireworks' mkBF/mkBS/mkBT embed the entire editable source
#         document, and prVw carries a full preview image.
#
# Why APP14 "Adobe" survives: it is not metadata, it declares the colour
# transform (YCbCr vs RGB vs YCCK). Dropping it turns some CMYK and RGB JPEGs
# into colour-inverted garbage. It is a 12-byte codec record written by many
# encoders, not a Photoshop watermark.
#
# Symlinks are resolved and the real file is rewritten, so cleaning a link no
# longer leaves the target dirty. A file with more than one hard link is
# rewritten in place rather than replaced, so every name sees the clean data —
# that write is not atomic, which is the honest trade for not silently breaking
# the link.
#
# Files are otherwise rewritten via a temp file in the same directory and moved
# into place, so an interrupted run leaves the original intact. A file that
# fails to parse is skipped, never half-written.
#
# Deliberately not handled: GIF, TIFF, HEIC, WebP, video. They need a real
# metadata library; this script would rather refuse than corrupt them. Install
# exiftool for those.

die() { >&2 echo "sz-rm-meta: $*"; exit 1; }
warn() { >&2 echo "sz-rm-meta: warning: $*"; }

list_only=false
out_dir=""
keep_icc=false
keep_xattr=false
destructive=false
rename=false
quality=85
scale=97
quiet=false
files=()

usage() {
    cat <<'USAGE'
sz-rm-meta.sh [options] FILE...   strip metadata from JPEG/PNG

  -l, --list         report what would be removed; change nothing
  -o, --out DIR      write cleaned copies into DIR instead of editing in place
      --keep-icc     keep the embedded ICC colour profile (stripped by default)
      --keep-xattr   leave extended attributes alone (macOS keeps the download
                     URL in kMDItemWhereFroms)
  -d, --destructive  also re-encode the pixels: breaks the bitstream fingerprint
                     linking this copy to the source. Needs ImageMagick.
      --quality N    JPEG quality for --destructive (default 85)
      --scale PCT    resize percentage for --destructive (default 97)
      --rename       rename to a random name, keeping the extension
  -q, --quiet        only report failures

Default mode is lossless: pixels stay bit-identical, which also means the file
still matches the source under reverse-image search or perceptual hashing.
Use --destructive when the goal is that a copy cannot be linked to its origin.
USAGE
}

while [ "$#" -gt 0 ]; do
    case "$1" in
        -l|--list)        list_only=true; shift ;;
        -o|--out)         [ "$#" -ge 2 ] || die "--out needs a directory"; out_dir="$2"; shift 2 ;;
        --keep-icc)       keep_icc=true; shift ;;
        --keep-xattr)     keep_xattr=true; shift ;;
        -d|--destructive) destructive=true; shift ;;
        --quality)        [ "$#" -ge 2 ] || die "--quality needs a number"; quality="$2"; shift 2 ;;
        --scale)          [ "$#" -ge 2 ] || die "--scale needs a number"; scale="$2"; shift 2 ;;
        --rename)         rename=true; shift ;;
        -q|--quiet)       quiet=true; shift ;;
        -h|--help)        usage; exit 0 ;;
        --)               shift; files+=("$@"); break ;;
        -*)               die "unknown option '$1' (try --help)" ;;
        *)                files+=("$1"); shift ;;
    esac
done

[ "${#files[@]}" -gt 0 ] || die "no files given (try --help)"
command -v python3 >/dev/null || die "python3 not found"

case "${quality}" in ''|*[!0-9]*) die "--quality must be a number" ;; esac
case "${scale}" in ''|*[!0-9]*) die "--scale must be a number" ;; esac
if [ "${quality}" -lt 1 ] || [ "${quality}" -gt 100 ]; then die "--quality must be 1-100"; fi
if [ "${scale}" -lt 1 ] || [ "${scale}" -gt 100 ]; then die "--scale must be 1-100"; fi

if [ -n "${out_dir}" ]; then
    ${list_only} && die "--list and --out are mutually exclusive"
    mkdir -p "${out_dir}"
fi

magick_cmd=""
if ${destructive}; then
    ${list_only} && die "--list and --destructive are mutually exclusive"
    if command -v magick >/dev/null 2>&1; then
        magick_cmd="magick"
    elif command -v convert >/dev/null 2>&1; then
        magick_cmd="convert"
    else
        die "--destructive needs ImageMagick (brew install imagemagick / apt install imagemagick)"
    fi
fi

if ${rename} && ${list_only}; then
    die "--list and --rename are mutually exclusive"
fi

# Whether this host can inspect xattrs at all. Without a tool we must say so
# rather than report a reassuring zero: Linux browsers write the download URL to
# user.xdg.origin.url, the direct analogue of macOS kMDItemWhereFroms.
xattr_tool=""
if command -v xattr >/dev/null 2>&1; then
    xattr_tool="macos"
elif command -v getfattr >/dev/null 2>&1 && command -v setfattr >/dev/null 2>&1; then
    xattr_tool="linux"
fi
if [ -z "${xattr_tool}" ] && ! ${keep_xattr} && [ -z "${out_dir}" ]; then
    warn "no xattr/setfattr on this host; extended attributes are NOT being checked or cleared"
fi

# Clear the extended attributes that carry provenance. com.apple.provenance is
# system-managed and cannot be removed - it holds no information about the
# image, so it is excluded from the count and left alone.
clear_xattrs() {
    local f="$1" n=0 a
    case "${xattr_tool}" in
        macos)
            n=$(xattr "${f}" 2>/dev/null | grep -cv '^com\.apple\.provenance$' || true)
            if [ "${n}" -gt 0 ]; then
                xattr -c "${f}" 2>/dev/null || true
            fi
            ;;
        linux)
            while read -r a; do
                [ -n "${a}" ] || continue
                if setfattr -x "${a}" "${f}" 2>/dev/null; then
                    n=$((n + 1))
                fi
            done < <(getfattr --absolute-names -m '^user\.' "${f}" 2>/dev/null |
                         grep '^user\.' || true)
            ;;
    esac
    echo "${n}"
}

count_xattrs() {
    case "${xattr_tool}" in
        macos) xattr "$1" 2>/dev/null | grep -cv '^com\.apple\.provenance$' || true ;;
        linux) getfattr -m '^user\.' "$1" 2>/dev/null | grep -c '^user\.' || true ;;
        *)     echo 0 ;;
    esac
}

strip_one() {
    python3 - "$1" "$2" "$3" "$4" <<'PY'
import os, shutil, struct, sys, tempfile

src, dst, mode, keep_icc = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] == "true"
listing = mode == "list"

def fail(msg):
    sys.stderr.write("sz-rm-meta: %s: %s\n" % (os.path.basename(src), msg))
    sys.exit(2)

# Follow symlinks: replacing the link node would leave the real file dirty while
# reporting success, and would silently turn the link into a regular file.
src = os.path.realpath(src)
in_place = os.path.realpath(dst) == src
if in_place:
    dst = src

d = open(src, "rb").read()
removed = {}   # label -> [count, bytes], insertion-ordered

def note(label, n):
    slot = removed.setdefault(label, [0, 0])
    slot[0] += 1
    slot[1] += n

def summary():
    parts = []
    for label, (count, size) in removed.items():
        parts.append("%s%s %dB" % (label, " x%d" % count if count > 1 else "", size))
    return ", ".join(parts)

def jpeg_label(marker, body):
    if marker == 0xFE:
        return "comment"
    if body.startswith(b"Exif\x00"):
        return "EXIF"
    if body.startswith(b"http://ns.adobe.com/xap"):
        return "XMP"
    if body.startswith(b"http://ns.adobe.com/xmp/extension"):
        return "XMP-extended"
    if body.startswith(b"Photoshop 3.0"):
        return "IPTC/8BIM"
    if body.startswith(b"ICC_PROFILE"):
        return "ICC"
    if body.startswith(b"Ducky"):
        return "Ducky"
    if body.startswith(b"http://ns.google.com"):
        return "Google-XMP"
    if body.startswith(b"MPF\x00"):
        return "MPF"
    return "APP%d" % (marker - 0xE0)

def strip_jpeg(d):
    out = bytearray(d[:2])
    i, n = 2, len(d)
    saw_eoi = False
    while i < n - 1:
        if d[i] != 0xFF:
            fail("malformed JPEG: expected a marker at offset %d" % i)
        m = d[i + 1]
        if m == 0xFF:                       # fill byte
            out += d[i:i + 1]; i += 1; continue
        if m == 0xD9:                       # EOI
            out += d[i:i + 2]; i += 2; saw_eoi = True; break
        if m == 0x01 or 0xD0 <= m <= 0xD7:  # TEM / RSTn: no payload
            out += d[i:i + 2]; i += 2; continue
        if i + 4 > n:
            fail("truncated JPEG segment header")
        ln = struct.unpack(">H", d[i + 2:i + 4])[0]
        if ln < 2 or i + 2 + ln > n:
            fail("truncated JPEG segment (marker FF%02X)" % m)
        body = d[i + 4:i + 2 + ln]
        keep = True
        if 0xE0 <= m <= 0xEF or m == 0xFE:
            label = jpeg_label(m, body)
            if m == 0xE0 and body.startswith(b"JFIF\x00"):
                # Density header. Its last two fields are thumbnail dimensions;
                # a non-zero pair means an embedded RGB preview of the image
                # rides along, so drop the segment rather than keep the preview.
                keep = not (len(body) >= 14 and body[12] and body[13])
                if not keep:
                    note("JFIF-thumbnail", ln + 2)
            elif m == 0xE0 and body.startswith(b"JFXX\x00"):
                keep = False                # exists only to carry a thumbnail
                note("JFXX-thumbnail", ln + 2)
            elif m == 0xEE and body.startswith(b"Adobe"):
                keep = True                 # colour transform: a codec parameter
            elif label == "ICC" and keep_icc:
                keep = True
            else:
                keep = False
                note(label, ln + 2)
        if keep:
            out += d[i:i + 2 + ln]
        i += 2 + ln
        if m == 0xDA:                       # entropy-coded scan follows
            j = i
            while j < n - 1:
                if d[j] != 0xFF:
                    j += 1; continue
                nx = d[j + 1]
                if nx == 0x00 or 0xD0 <= nx <= 0xD7:
                    j += 2; continue        # byte stuffing / restart marker
                if nx == 0xFF:
                    j += 1; continue        # fill byte
                break
            out += d[i:j]
            i = j
    if not saw_eoi:
        fail("no EOI marker found; refusing to write a truncated file")
    if i < n:
        note("trailer", n - i)              # appended thumbnails, depth maps, junk
    return bytes(out)

# Allowlist, not denylist: anything not named here is dropped. Covers the image
# itself, transparency, colour rendering, and APNG animation. Private chunks
# (mkBF/prVw/cmOD and friends) fall through to the drop path by construction.
PNG_KEEP = {
    b"IHDR", b"PLTE", b"IDAT", b"IEND",             # critical
    b"tRNS", b"gAMA", b"cHRM", b"sRGB", b"sBIT",    # colour / transparency
    b"bKGD", b"pHYs", b"hIST", b"sPLT",             # rendering hints
    b"acTL", b"fcTL", b"fdAT",                      # APNG: dropping these
}                                                   # would flatten the animation

def strip_png(d):
    out = bytearray(d[:8])
    i, n = 8, len(d)
    saw_end = False
    while i + 8 <= n:
        ln = struct.unpack(">I", d[i:i + 4])[0]
        typ = d[i + 4:i + 8]
        end = i + 12 + ln
        if end > n:
            fail("truncated PNG chunk '%s'" % typ.decode("ascii", "replace"))
        keep = typ in PNG_KEEP or (typ == b"iCCP" and keep_icc)
        if keep:
            out += d[i:end]
        else:
            note(typ.decode("ascii", "replace"), ln + 12)
        i = end
        if typ == b"IEND":
            saw_end = True
            break
    if not saw_end:
        fail("no IEND chunk found; refusing to write a truncated file")
    if i < n:
        note("trailer", n - i)
    return bytes(out)

if d[:2] == b"\xff\xd8":
    cleaned = strip_jpeg(d)
elif d[:8] == b"\x89PNG\r\n\x1a\n":
    cleaned = strip_png(d)
else:
    fail("unsupported format (only JPEG and PNG; use exiftool for others)")

links = os.stat(src).st_nlink if in_place else 1

if not listing and (removed or not in_place):
    if in_place and links > 1:
        # Replacing the inode would clean this name and leave every other hard
        # link holding the original metadata. Rewrite the inode instead so all
        # names see the clean data, accepting a non-atomic write.
        with open(dst, "r+b") as fh:
            fh.write(cleaned)
            fh.truncate()
    else:
        # Write beside the destination so the rename is atomic on the same volume.
        dst_dir = os.path.dirname(os.path.abspath(dst)) or "."
        fd, tmp = tempfile.mkstemp(dir=dst_dir, prefix=".sz-rm-meta.")
        try:
            with os.fdopen(fd, "wb") as fh:
                fh.write(cleaned)
            shutil.copymode(src, tmp)
            os.replace(tmp, dst)
        except BaseException:
            if os.path.exists(tmp):
                os.unlink(tmp)
            raise

print("%d\t%d\t%d\t%s" % (len(d), len(cleaned), links, summary()))
PY
}

# Re-encode into a temp file: new quantization tables, new bitstream, slightly
# different dimensions. -strip drops what ImageMagick knows about; the lossless
# pass afterwards catches what it leaves behind (it writes its own tEXt/date
# chunks into PNG, for one).
reencode() {
    local src="$1" tmp="$2"
    "${magick_cmd}" "${src}" -strip -resize "${scale}%" -quality "${quality}" \
        -interlace none "${tmp}" 2>/dev/null
}

random_name() {
    local ext="${1##*.}"
    local stem
    stem=$(LC_ALL=C tr -dc 'a-z0-9' </dev/urandom | head -c 12)
    if [ "${ext}" = "$1" ]; then
        echo "${stem}"
    else
        echo "${stem}.${ext}"
    fi
}

status=0
for f in "${files[@]}"; do
    [ -f "${f}" ] || { >&2 echo "sz-rm-meta: not a file: ${f}"; status=1; continue; }

    orig_size=$(python3 -c 'import os,sys; print(os.path.getsize(sys.argv[1]))' "${f}")

    dst="${f}"
    [ -n "${out_dir}" ] && dst="${out_dir%/}/$(basename "${f}")"

    # Count before rewriting: the rewrite replaces the inode, which drops the
    # xattrs on its own, so counting afterwards would always report zero. In
    # --out mode the source is never touched and the fresh copy starts with no
    # xattrs, so there is nothing to clear or report.
    xa=0
    if ! ${keep_xattr} && [ -z "${out_dir}" ]; then
        xa=$(count_xattrs "${f}")
    fi

    src="${f}"
    tmp_reenc=""
    if ${destructive}; then
        tmp_reenc="$(dirname "${dst}")/.sz-rm-meta-reenc.$$.$(basename "${f}")"
        if ! reencode "${f}" "${tmp_reenc}"; then
            rm -f "${tmp_reenc}"
            >&2 echo "sz-rm-meta: ${f}: re-encode failed"
            status=1; continue
        fi
        src="${tmp_reenc}"
    fi

    mode="strip"; ${list_only} && mode="list"
    if ! result=$(strip_one "${src}" "${dst}" "${mode}" "${keep_icc}"); then
        [ -n "${tmp_reenc}" ] && rm -f "${tmp_reenc}"
        status=1; continue
    fi
    if [ -n "${tmp_reenc}" ]; then
        rm -f "${tmp_reenc}"
    fi

    # Still needed for a file that was already clean, and for the hard-link path
    # where the inode survives the rewrite along with its xattrs.
    if [ "${xa}" -gt 0 ] && ! ${list_only}; then
        clear_xattrs "${dst}" >/dev/null
    fi

    after=$(echo "${result}" | cut -f2)
    links=$(echo "${result}" | cut -f3)
    removed=$(echo "${result}" | cut -f4)

    final="${dst}"
    if ${rename} && ! ${list_only}; then
        final="$(dirname "${dst}")/$(random_name "$(basename "${dst}")")"
        mv -n "${dst}" "${final}" || { >&2 echo "sz-rm-meta: rename failed: ${dst}"; status=1; final="${dst}"; }
    fi

    ${quiet} && continue

    detail="${removed}"
    if [ "${xa}" -gt 0 ]; then
        plural=s; [ "${xa}" -eq 1 ] && plural=
        detail="${detail:+${detail}, }${xa} xattr${plural}"
    fi
    if [ -z "${detail}" ] && ! ${destructive}; then
        echo "${f}: already clean"
    elif ${list_only}; then
        echo "${f}: would remove ${detail} (${orig_size} -> ${after} bytes)"
    else
        if ${destructive}; then
            action="re-encoded q${quality} @${scale}%${detail:+, removed ${detail}}"
        else
            action="removed ${detail}"
        fi
        line="${f}: ${action} (${orig_size} -> ${after} bytes)"
        [ "${links}" -gt 1 ] && line="${line} [${links} hard links, rewritten in place]"
        [ "${final}" != "${dst}" ] && line="${line} -> $(basename "${final}")"
        echo "${line}"
    fi
done

exit "${status}"