diff options
Diffstat (limited to 'bin/sz-rm-meta.sh')
| -rwxr-xr-x | bin/sz-rm-meta.sh | 477 |
1 files changed, 477 insertions, 0 deletions
diff --git a/bin/sz-rm-meta.sh b/bin/sz-rm-meta.sh new file mode 100755 index 0000000..5618dc2 --- /dev/null +++ b/bin/sz-rm-meta.sh @@ -0,0 +1,477 @@ +#!/usr/bin/env bash +set -euo pipefail + +# sz-rm-meta.sh — strip metadata from image files. +# +# sz-rm-meta.sh [options] FILE... +# +# -l, --list report what would be removed; change nothing +# -o, --out DIR write cleaned copies into DIR instead of editing in place +# --keep-icc keep the embedded ICC colour profile (stripped by default) +# --keep-xattr leave extended attributes alone (macOS: kMDItemWhereFroms, +# com.apple.quarantine — these record the download URL) +# -d, --destructive re-encode the pixels as well (see "Two modes" below) +# --quality N JPEG quality for --destructive (default 85) +# --scale PCT resize percentage for --destructive (default 97) +# --rename rename the file to a random name, keeping the extension +# -q, --quiet only report failures +# +# Two modes, because "no metadata" and "not traceable" are different jobs: +# +# default Lossless. Rewrites the container, copies the compressed image +# data byte for byte. Pixels stay bit-identical, so the file is +# still matchable against the source by perceptual hash, and the +# quantization/Huffman tables still fingerprint the encoder that +# produced it. Right for sanitising an archive you keep. +# +# --destructive Re-encodes: resize by --scale, re-compress at --quality, fresh +# quantization tables, then run the lossless pass over the +# result. Breaks bitstream-level linkage to the source copy. +# Does NOT defeat reverse-image search or perceptual hashing — +# nothing that preserves what the picture looks like can. Needs +# ImageMagick. +# +# What the lossless pass removes, per format: +# JPEG every APPn segment except JFIF (APP0) and Adobe (APP14), all COM +# comments, and any trailer appended after the EOI marker. That covers +# EXIF (camera, GPS, serial, timestamps), XMP, IPTC and the Photoshop +# 8BIM block (by-line, credit, document ancestry). +# PNG everything outside an allowlist of image and colour chunks. An +# allowlist rather than a denylist because private chunks are the worst +# offenders: Fireworks' mkBF/mkBS/mkBT embed the entire editable source +# document, and prVw carries a full preview image. +# +# Why APP14 "Adobe" survives: it is not metadata, it declares the colour +# transform (YCbCr vs RGB vs YCCK). Dropping it turns some CMYK and RGB JPEGs +# into colour-inverted garbage. It is a 12-byte codec record written by many +# encoders, not a Photoshop watermark. +# +# Symlinks are resolved and the real file is rewritten, so cleaning a link no +# longer leaves the target dirty. A file with more than one hard link is +# rewritten in place rather than replaced, so every name sees the clean data — +# that write is not atomic, which is the honest trade for not silently breaking +# the link. +# +# Files are otherwise rewritten via a temp file in the same directory and moved +# into place, so an interrupted run leaves the original intact. A file that +# fails to parse is skipped, never half-written. +# +# Deliberately not handled: GIF, TIFF, HEIC, WebP, video. They need a real +# metadata library; this script would rather refuse than corrupt them. Install +# exiftool for those. + +die() { >&2 echo "sz-rm-meta: $*"; exit 1; } +warn() { >&2 echo "sz-rm-meta: warning: $*"; } + +list_only=false +out_dir="" +keep_icc=false +keep_xattr=false +destructive=false +rename=false +quality=85 +scale=97 +quiet=false +files=() + +usage() { + cat <<'USAGE' +sz-rm-meta.sh [options] FILE... strip metadata from JPEG/PNG + + -l, --list report what would be removed; change nothing + -o, --out DIR write cleaned copies into DIR instead of editing in place + --keep-icc keep the embedded ICC colour profile (stripped by default) + --keep-xattr leave extended attributes alone (macOS keeps the download + URL in kMDItemWhereFroms) + -d, --destructive also re-encode the pixels: breaks the bitstream fingerprint + linking this copy to the source. Needs ImageMagick. + --quality N JPEG quality for --destructive (default 85) + --scale PCT resize percentage for --destructive (default 97) + --rename rename to a random name, keeping the extension + -q, --quiet only report failures + +Default mode is lossless: pixels stay bit-identical, which also means the file +still matches the source under reverse-image search or perceptual hashing. +Use --destructive when the goal is that a copy cannot be linked to its origin. +USAGE +} + +while [ "$#" -gt 0 ]; do + case "$1" in + -l|--list) list_only=true; shift ;; + -o|--out) [ "$#" -ge 2 ] || die "--out needs a directory"; out_dir="$2"; shift 2 ;; + --keep-icc) keep_icc=true; shift ;; + --keep-xattr) keep_xattr=true; shift ;; + -d|--destructive) destructive=true; shift ;; + --quality) [ "$#" -ge 2 ] || die "--quality needs a number"; quality="$2"; shift 2 ;; + --scale) [ "$#" -ge 2 ] || die "--scale needs a number"; scale="$2"; shift 2 ;; + --rename) rename=true; shift ;; + -q|--quiet) quiet=true; shift ;; + -h|--help) usage; exit 0 ;; + --) shift; files+=("$@"); break ;; + -*) die "unknown option '$1' (try --help)" ;; + *) files+=("$1"); shift ;; + esac +done + +[ "${#files[@]}" -gt 0 ] || die "no files given (try --help)" +command -v python3 >/dev/null || die "python3 not found" + +case "${quality}" in ''|*[!0-9]*) die "--quality must be a number" ;; esac +case "${scale}" in ''|*[!0-9]*) die "--scale must be a number" ;; esac +if [ "${quality}" -lt 1 ] || [ "${quality}" -gt 100 ]; then die "--quality must be 1-100"; fi +if [ "${scale}" -lt 1 ] || [ "${scale}" -gt 100 ]; then die "--scale must be 1-100"; fi + +if [ -n "${out_dir}" ]; then + ${list_only} && die "--list and --out are mutually exclusive" + mkdir -p "${out_dir}" +fi + +magick_cmd="" +if ${destructive}; then + ${list_only} && die "--list and --destructive are mutually exclusive" + if command -v magick >/dev/null 2>&1; then + magick_cmd="magick" + elif command -v convert >/dev/null 2>&1; then + magick_cmd="convert" + else + die "--destructive needs ImageMagick (brew install imagemagick / apt install imagemagick)" + fi +fi + +if ${rename} && ${list_only}; then + die "--list and --rename are mutually exclusive" +fi + +# Whether this host can inspect xattrs at all. Without a tool we must say so +# rather than report a reassuring zero: Linux browsers write the download URL to +# user.xdg.origin.url, the direct analogue of macOS kMDItemWhereFroms. +xattr_tool="" +if command -v xattr >/dev/null 2>&1; then + xattr_tool="macos" +elif command -v getfattr >/dev/null 2>&1 && command -v setfattr >/dev/null 2>&1; then + xattr_tool="linux" +fi +if [ -z "${xattr_tool}" ] && ! ${keep_xattr} && [ -z "${out_dir}" ]; then + warn "no xattr/setfattr on this host; extended attributes are NOT being checked or cleared" +fi + +# Clear the extended attributes that carry provenance. com.apple.provenance is +# system-managed and cannot be removed - it holds no information about the +# image, so it is excluded from the count and left alone. +clear_xattrs() { + local f="$1" n=0 a + case "${xattr_tool}" in + macos) + n=$(xattr "${f}" 2>/dev/null | grep -cv '^com\.apple\.provenance$' || true) + if [ "${n}" -gt 0 ]; then + xattr -c "${f}" 2>/dev/null || true + fi + ;; + linux) + while read -r a; do + [ -n "${a}" ] || continue + if setfattr -x "${a}" "${f}" 2>/dev/null; then + n=$((n + 1)) + fi + done < <(getfattr --absolute-names -m '^user\.' "${f}" 2>/dev/null | + grep '^user\.' || true) + ;; + esac + echo "${n}" +} + +count_xattrs() { + case "${xattr_tool}" in + macos) xattr "$1" 2>/dev/null | grep -cv '^com\.apple\.provenance$' || true ;; + linux) getfattr -m '^user\.' "$1" 2>/dev/null | grep -c '^user\.' || true ;; + *) echo 0 ;; + esac +} + +strip_one() { + python3 - "$1" "$2" "$3" "$4" <<'PY' +import os, shutil, struct, sys, tempfile + +src, dst, mode, keep_icc = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] == "true" +listing = mode == "list" + +def fail(msg): + sys.stderr.write("sz-rm-meta: %s: %s\n" % (os.path.basename(src), msg)) + sys.exit(2) + +# Follow symlinks: replacing the link node would leave the real file dirty while +# reporting success, and would silently turn the link into a regular file. +src = os.path.realpath(src) +in_place = os.path.realpath(dst) == src +if in_place: + dst = src + +d = open(src, "rb").read() +removed = {} # label -> [count, bytes], insertion-ordered + +def note(label, n): + slot = removed.setdefault(label, [0, 0]) + slot[0] += 1 + slot[1] += n + +def summary(): + parts = [] + for label, (count, size) in removed.items(): + parts.append("%s%s %dB" % (label, " x%d" % count if count > 1 else "", size)) + return ", ".join(parts) + +def jpeg_label(marker, body): + if marker == 0xFE: + return "comment" + if body.startswith(b"Exif\x00"): + return "EXIF" + if body.startswith(b"http://ns.adobe.com/xap"): + return "XMP" + if body.startswith(b"http://ns.adobe.com/xmp/extension"): + return "XMP-extended" + if body.startswith(b"Photoshop 3.0"): + return "IPTC/8BIM" + if body.startswith(b"ICC_PROFILE"): + return "ICC" + if body.startswith(b"Ducky"): + return "Ducky" + if body.startswith(b"http://ns.google.com"): + return "Google-XMP" + if body.startswith(b"MPF\x00"): + return "MPF" + return "APP%d" % (marker - 0xE0) + +def strip_jpeg(d): + out = bytearray(d[:2]) + i, n = 2, len(d) + saw_eoi = False + while i < n - 1: + if d[i] != 0xFF: + fail("malformed JPEG: expected a marker at offset %d" % i) + m = d[i + 1] + if m == 0xFF: # fill byte + out += d[i:i + 1]; i += 1; continue + if m == 0xD9: # EOI + out += d[i:i + 2]; i += 2; saw_eoi = True; break + if m == 0x01 or 0xD0 <= m <= 0xD7: # TEM / RSTn: no payload + out += d[i:i + 2]; i += 2; continue + if i + 4 > n: + fail("truncated JPEG segment header") + ln = struct.unpack(">H", d[i + 2:i + 4])[0] + if ln < 2 or i + 2 + ln > n: + fail("truncated JPEG segment (marker FF%02X)" % m) + body = d[i + 4:i + 2 + ln] + keep = True + if 0xE0 <= m <= 0xEF or m == 0xFE: + label = jpeg_label(m, body) + if m == 0xE0 and body.startswith(b"JFIF\x00"): + # Density header. Its last two fields are thumbnail dimensions; + # a non-zero pair means an embedded RGB preview of the image + # rides along, so drop the segment rather than keep the preview. + keep = not (len(body) >= 14 and body[12] and body[13]) + if not keep: + note("JFIF-thumbnail", ln + 2) + elif m == 0xE0 and body.startswith(b"JFXX\x00"): + keep = False # exists only to carry a thumbnail + note("JFXX-thumbnail", ln + 2) + elif m == 0xEE and body.startswith(b"Adobe"): + keep = True # colour transform: a codec parameter + elif label == "ICC" and keep_icc: + keep = True + else: + keep = False + note(label, ln + 2) + if keep: + out += d[i:i + 2 + ln] + i += 2 + ln + if m == 0xDA: # entropy-coded scan follows + j = i + while j < n - 1: + if d[j] != 0xFF: + j += 1; continue + nx = d[j + 1] + if nx == 0x00 or 0xD0 <= nx <= 0xD7: + j += 2; continue # byte stuffing / restart marker + if nx == 0xFF: + j += 1; continue # fill byte + break + out += d[i:j] + i = j + if not saw_eoi: + fail("no EOI marker found; refusing to write a truncated file") + if i < n: + note("trailer", n - i) # appended thumbnails, depth maps, junk + return bytes(out) + +# Allowlist, not denylist: anything not named here is dropped. Covers the image +# itself, transparency, colour rendering, and APNG animation. Private chunks +# (mkBF/prVw/cmOD and friends) fall through to the drop path by construction. +PNG_KEEP = { + b"IHDR", b"PLTE", b"IDAT", b"IEND", # critical + b"tRNS", b"gAMA", b"cHRM", b"sRGB", b"sBIT", # colour / transparency + b"bKGD", b"pHYs", b"hIST", b"sPLT", # rendering hints + b"acTL", b"fcTL", b"fdAT", # APNG: dropping these +} # would flatten the animation + +def strip_png(d): + out = bytearray(d[:8]) + i, n = 8, len(d) + saw_end = False + while i + 8 <= n: + ln = struct.unpack(">I", d[i:i + 4])[0] + typ = d[i + 4:i + 8] + end = i + 12 + ln + if end > n: + fail("truncated PNG chunk '%s'" % typ.decode("ascii", "replace")) + keep = typ in PNG_KEEP or (typ == b"iCCP" and keep_icc) + if keep: + out += d[i:end] + else: + note(typ.decode("ascii", "replace"), ln + 12) + i = end + if typ == b"IEND": + saw_end = True + break + if not saw_end: + fail("no IEND chunk found; refusing to write a truncated file") + if i < n: + note("trailer", n - i) + return bytes(out) + +if d[:2] == b"\xff\xd8": + cleaned = strip_jpeg(d) +elif d[:8] == b"\x89PNG\r\n\x1a\n": + cleaned = strip_png(d) +else: + fail("unsupported format (only JPEG and PNG; use exiftool for others)") + +links = os.stat(src).st_nlink if in_place else 1 + +if not listing and (removed or not in_place): + if in_place and links > 1: + # Replacing the inode would clean this name and leave every other hard + # link holding the original metadata. Rewrite the inode instead so all + # names see the clean data, accepting a non-atomic write. + with open(dst, "r+b") as fh: + fh.write(cleaned) + fh.truncate() + else: + # Write beside the destination so the rename is atomic on the same volume. + dst_dir = os.path.dirname(os.path.abspath(dst)) or "." + fd, tmp = tempfile.mkstemp(dir=dst_dir, prefix=".sz-rm-meta.") + try: + with os.fdopen(fd, "wb") as fh: + fh.write(cleaned) + shutil.copymode(src, tmp) + os.replace(tmp, dst) + except BaseException: + if os.path.exists(tmp): + os.unlink(tmp) + raise + +print("%d\t%d\t%d\t%s" % (len(d), len(cleaned), links, summary())) +PY +} + +# Re-encode into a temp file: new quantization tables, new bitstream, slightly +# different dimensions. -strip drops what ImageMagick knows about; the lossless +# pass afterwards catches what it leaves behind (it writes its own tEXt/date +# chunks into PNG, for one). +reencode() { + local src="$1" tmp="$2" + "${magick_cmd}" "${src}" -strip -resize "${scale}%" -quality "${quality}" \ + -interlace none "${tmp}" 2>/dev/null +} + +random_name() { + local ext="${1##*.}" + local stem + stem=$(LC_ALL=C tr -dc 'a-z0-9' </dev/urandom | head -c 12) + if [ "${ext}" = "$1" ]; then + echo "${stem}" + else + echo "${stem}.${ext}" + fi +} + +status=0 +for f in "${files[@]}"; do + [ -f "${f}" ] || { >&2 echo "sz-rm-meta: not a file: ${f}"; status=1; continue; } + + orig_size=$(python3 -c 'import os,sys; print(os.path.getsize(sys.argv[1]))' "${f}") + + dst="${f}" + [ -n "${out_dir}" ] && dst="${out_dir%/}/$(basename "${f}")" + + # Count before rewriting: the rewrite replaces the inode, which drops the + # xattrs on its own, so counting afterwards would always report zero. In + # --out mode the source is never touched and the fresh copy starts with no + # xattrs, so there is nothing to clear or report. + xa=0 + if ! ${keep_xattr} && [ -z "${out_dir}" ]; then + xa=$(count_xattrs "${f}") + fi + + src="${f}" + tmp_reenc="" + if ${destructive}; then + tmp_reenc="$(dirname "${dst}")/.sz-rm-meta-reenc.$$.$(basename "${f}")" + if ! reencode "${f}" "${tmp_reenc}"; then + rm -f "${tmp_reenc}" + >&2 echo "sz-rm-meta: ${f}: re-encode failed" + status=1; continue + fi + src="${tmp_reenc}" + fi + + mode="strip"; ${list_only} && mode="list" + if ! result=$(strip_one "${src}" "${dst}" "${mode}" "${keep_icc}"); then + [ -n "${tmp_reenc}" ] && rm -f "${tmp_reenc}" + status=1; continue + fi + if [ -n "${tmp_reenc}" ]; then + rm -f "${tmp_reenc}" + fi + + # Still needed for a file that was already clean, and for the hard-link path + # where the inode survives the rewrite along with its xattrs. + if [ "${xa}" -gt 0 ] && ! ${list_only}; then + clear_xattrs "${dst}" >/dev/null + fi + + after=$(echo "${result}" | cut -f2) + links=$(echo "${result}" | cut -f3) + removed=$(echo "${result}" | cut -f4) + + final="${dst}" + if ${rename} && ! ${list_only}; then + final="$(dirname "${dst}")/$(random_name "$(basename "${dst}")")" + mv -n "${dst}" "${final}" || { >&2 echo "sz-rm-meta: rename failed: ${dst}"; status=1; final="${dst}"; } + fi + + ${quiet} && continue + + detail="${removed}" + if [ "${xa}" -gt 0 ]; then + plural=s; [ "${xa}" -eq 1 ] && plural= + detail="${detail:+${detail}, }${xa} xattr${plural}" + fi + if [ -z "${detail}" ] && ! ${destructive}; then + echo "${f}: already clean" + elif ${list_only}; then + echo "${f}: would remove ${detail} (${orig_size} -> ${after} bytes)" + else + if ${destructive}; then + action="re-encoded q${quality} @${scale}%${detail:+, removed ${detail}}" + else + action="removed ${detail}" + fi + line="${f}: ${action} (${orig_size} -> ${after} bytes)" + [ "${links}" -gt 1 ] && line="${line} [${links} hard links, rewritten in place]" + [ "${final}" != "${dst}" ] && line="${line} -> $(basename "${final}")" + echo "${line}" + fi +done + +exit "${status}" |