Skip to content

multiple apps with the same root directory #15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

multiple apps with the same root directory #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,19 @@ now add this to your settings.py ('app' is your project name where models.py is
}
}

if you have multiple apps with the same media root like:

ORPHANED_APPS_MEDIABASE_DIRS = {
'app1': {
'root': MEDIA_ROOT,
},
'app2': {
'root': MEDIA_ROOT,
},
}

django-orphaned will not delete files needed only in one of them.

**NOTE**: from version 0.4.2 you can define ''root'' as string or iterable (list, array)

the least to do is to run this command to show all orphaned files
Expand Down
227 changes: 127 additions & 100 deletions django_orphaned/management/commands/deleteorphaned.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@
from django_orphaned.app_settings import ORPHANED_APPS_MEDIABASE_DIRS
from itertools import chain
from optparse import make_option
from django.conf import settings

import os
import shutil
from django.conf import settings


class Command(BaseCommand):
help = "Delete all orphaned files"
Expand All @@ -15,106 +17,131 @@ class Command(BaseCommand):
)
option_list = BaseCommand.option_list + base_options

def _get_needed_files(self, app):
"""
collects all needed files for an app. goes through every field of each
model and detects FileFields and ImageFields
"""
needed_files = []
for model in ContentType.objects.filter(app_label=app):
mc = model.model_class()
if mc is None:
continue
fields = []
for field in mc._meta.fields:
if (field.get_internal_type() == 'FileField' or field.get_internal_type() == 'ImageField'):
fields.append(field.name)

# we have found a model with FileFields
if len(fields) > 0:
files = mc.objects.all().values_list(*fields)
needed_files.extend([os.path.join(settings.MEDIA_ROOT, file) for file in filter(None, chain.from_iterable(files))])

return needed_files

def _get_media_files(self, app):
"""
collects all media files and empty directories for an app. respects
the 'skip' and 'exclude' rules provided by the app settings.
"""
skip = ORPHANED_APPS_MEDIABASE_DIRS[app].get('skip', ())
exclude = ORPHANED_APPS_MEDIABASE_DIRS[app].get('exclude', ())

# traverse root folder and store all files and empty directories
def should_skip(dir):
for skip_dir in skip:
if dir.startswith(skip_dir):
return True
return False

# process each root of the app
app_roots = ORPHANED_APPS_MEDIABASE_DIRS[app]['root']
if isinstance(app_roots, basestring): # backwards compatibility
app_roots = [app_roots]

all_files = []
possible_empty_dirs = []
for app_root in app_roots:
for root, dirs, files in os.walk(app_root):
if should_skip(root):
continue
if len(files) > 0:
for basename in files:
if basename not in exclude:
all_files.append(os.path.join(root, basename))
elif not os.path.samefile(root, app_root):
possible_empty_dirs.append(root)

# ignore empty dirs with subdirs + files
empty_dirs = []
for ed in possible_empty_dirs:
dont_delete = False
for files in all_files:
try:
if files.index(ed) == 0:
dont_delete = True
except ValueError:
pass
for skip_dir in skip:
try:
if (skip_dir.index(ed) == 0):
dont_delete = True
except ValueError:
pass
if not dont_delete:
empty_dirs.append(ed)

return all_files, empty_dirs

def handle(self, **options):
"""
goes through ever app given in the settings. if option 'info' is given,
nothing would be deleted.
"""
self.only_info = options.get('info')

all_files = []
needed_files = []
empty_dirs = []
total_freed_bytes = 0
total_freed = '0'

for app in ORPHANED_APPS_MEDIABASE_DIRS.keys():
if (ORPHANED_APPS_MEDIABASE_DIRS[app].has_key('root')):
needed_files = []
all_files = []
possible_empty_dirs = []
empty_dirs = []
total_freed_bytes = 0
total_freed = '0'
delete_files = []
skip = ORPHANED_APPS_MEDIABASE_DIRS[app].get('skip', ())
exclude = ORPHANED_APPS_MEDIABASE_DIRS[app].get('exclude', ())

for model in ContentType.objects.filter(app_label=app):
mc = model.model_class()
if mc is None:
continue
fields = []
for field in mc._meta.fields:
if (field.get_internal_type() == 'FileField' or field.get_internal_type() == 'ImageField'):
fields.append(field.name)

# we have found a model with FileFields
if len(fields) > 0:
files = mc.objects.all().values_list(*fields)
needed_files.extend([os.path.join(settings.MEDIA_ROOT, file) for file in filter(None, chain.from_iterable(files))])

# traverse root folder and store all files and empty directories
def should_skip(dir):
for skip_dir in skip:
if dir.startswith(skip_dir):
return True
return False

# process each root of the app
app_roots = ORPHANED_APPS_MEDIABASE_DIRS[app]['root']
if isinstance(app_roots, basestring): # backwards compatibility
app_roots = [app_roots]
for app_root in app_roots:
for root, dirs, files in os.walk(app_root):
if should_skip(root):
continue
if len(files) > 0:
for basename in files:
if basename not in exclude:
all_files.append(os.path.join(root, basename))
elif not os.path.samefile(root, app_root):
possible_empty_dirs.append(root)

# ignore empty dirs with subdirs + files
for ed in possible_empty_dirs:
dont_delete = False
for files in all_files:
try:
if files.index(ed) == 0:
dont_delete = True
except ValueError:
pass
for skip_dir in skip:
try:
if (skip_dir.index(ed) == 0):
dont_delete = True
except ValueError:
pass
if not dont_delete:
empty_dirs.append(ed)

# select deleted files (delete_files = all_files - needed_files)
aa = set(all_files)
delete_files = list(aa.difference(needed_files))
delete_files.sort()
empty_dirs.sort()
empty_dirs = set(empty_dirs) #remove possible duplicates

# to be freed
for df in delete_files:
total_freed_bytes += os.path.getsize(df)
total_freed = "%0.1f MB" % (total_freed_bytes/(1024*1024.0))

# only show
if (self.only_info):
print "\r\n=== %s ===" % app
if len(empty_dirs) > 0:
print "\r\nFollowing empty dirs will be removed:\r\n"
for file in empty_dirs:
print " ", file

if len(delete_files) > 0:
print "\r\nFollowing files will be deleted:\r\n"
for file in delete_files:
print " ", file
print "\r\nTotally %s files will be deleted, and "\
"totally %s will be freed.\r\n" % (len(delete_files), total_freed)
else:
print "No files to delete!"
# DELETE NOW!
else:
for file in delete_files:
os.remove(file)
for dirs in empty_dirs:
shutil.rmtree(dirs, ignore_errors=True)
if ('root' in ORPHANED_APPS_MEDIABASE_DIRS[app]):
needed_files.extend(self._get_needed_files(app))
a, e = self._get_media_files(app)
all_files.extend(a)
empty_dirs.extend(e)

# select deleted files (delete_files = all_files - needed_files)
delete_files = sorted(set(all_files).difference(needed_files))
empty_dirs = sorted(set(empty_dirs)) # remove possible duplicates

# only show
if (self.only_info):
# to be freed
for df in delete_files:
total_freed_bytes += os.path.getsize(df)
total_freed = "%0.1f MB" % (total_freed_bytes / (1024 * 1024.0))

print "\r\n=== %s ===" % app
if len(empty_dirs) > 0:
print "\r\nFollowing empty dirs will be removed:\r\n"
for file in empty_dirs:
print " ", file

if len(delete_files) > 0:
print "\r\nFollowing files will be deleted:\r\n"
for file in delete_files:
print " ", file
print "\r\nTotally %s files will be deleted, and "\
"totally %s will be freed.\r\n" % (len(delete_files), total_freed)
else:
print "No files to delete!"

# otherwise delete
else:
for file in delete_files:
os.remove(file)
for dirs in empty_dirs:
shutil.rmtree(dirs, ignore_errors=True)