Python Script for Duplicate Image Remover

Ever bulk download images for a dataset and discover you have tons of duplicates? This simple Python script will use a very effective method of removing duplicate images in any given folder.

Here's how it's done:

1. Download script file I provided,

2. Right-Click and open with your favorite text editor. Modify line 6 to the destination of your images and Save.

3. Simply double-click the file and it will automatically detect and remove any duplicate images. An added bonus is it will always prioritize PNG files over all else(i.e. if you have a duplicate image in both JPG and PNG format it will delete the JPG).

A few notes:

• This script actually takes the pixel information of the images into account, it will actually scan the images and determine duplicates.

• For my paranoid friends out there, I will provide the script below, ask ChatGPT to check it so you have nothing to worry about!

import os

from PIL import Image

import imagehash

# Folder path to check for duplicates

folder_path = r"C:\Path\To\Your\Images"

# Dictionary to store image hashes

hashes = {}

# Function to compare and find duplicates, preferring PNG over JPG

def find_duplicates(folder):

for root, _, files in os.walk(folder):

for file in files:

file_path = os.path.join(root, file)

try:

# Open image and calculate its hash

with Image.open(file_path) as img:

img_hash = imagehash.average_hash(img)

# Check if the hash already exists

if img_hash in hashes:

existing_file_path = hashes[img_hash]

# Check file extensions to prefer PNG over JPG

if file_path.lower().endswith('.png') and not existing_file_path.lower().endswith('.png'):

# PNG is preferred, so remove the existing non-PNG duplicate

print(f"Removing non-PNG duplicate: {existing_file_path}")

os.remove(existing_file_path)

hashes[img_hash] = file_path # Update hash to point to the PNG version

elif not file_path.lower().endswith('.png') and existing_file_path.lower().endswith('.png'):

# Existing PNG should be kept, so remove the current duplicate

print(f"Removing non-PNG duplicate: {file_path}")

os.remove(file_path)

else:

# If both are PNG or neither, just remove the current file

print(f"Removing duplicate: {file_path}")

os.remove(file_path)

else:

hashes[img_hash] = file_path # Store the hash

except Exception as e:

print(f"Error processing {file_path}: {e}")

if name == "__main__":

find_duplicates(folder_path)