dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove scandir lib, use os.walk

Browse Source
This commit is contained in:
Ruud
2014-05-31 11:53:24 +02:00
parent 7c0870b6b8
commit a5ee362fc0
12 changed files with 16 additions and 857 deletions
Download Patch File Download Diff File Expand all files Collapse all files
couchpotato/core/_base/updater/main.py
+3 -4
View File
@@ -16,7 +16,6 @@ from couchpotato.core.plugins.base import Plugin
from couchpotato.environment import Env
from dateutil.parser import parse
from git.repository import LocalRepository
from scandir import scandir
import version
from six.moves import filter
@@ -182,7 +181,7 @@ class BaseUpdater(Plugin):
def deletePyc(self, only_excess = True, show_logs = True):
for root, dirs, files in scandir.walk(Env.get('app_dir')):
for root, dirs, files in os.walk(Env.get('app_dir')):
pyc_files = filter(lambda filename: filename.endswith('.pyc'), files)
py_files = set(filter(lambda filename: filename.endswith('.py'), files))
@@ -329,11 +328,11 @@ class SourceUpdater(BaseUpdater):
# Get list of files we want to overwrite
self.deletePyc()
existing_files = []
for root, subfiles, filenames in scandir.walk(app_dir):
for root, subfiles, filenames in os.walk(app_dir):
for filename in filenames:
existing_files.append(os.path.join(root, filename))
for root, subfiles, filenames in scandir.walk(path):
for root, subfiles, filenames in os.walk(path):
for filename in filenames:
fromfile = os.path.join(root, filename)
tofile = os.path.join(app_dir, fromfile.replace(path + os.path.sep, ''))
couchpotato/core/downloaders/rtorrent_.py
+1 -2
View File
@@ -12,7 +12,6 @@ from couchpotato.core.helpers.variable import cleanHost, splitString
from couchpotato.core.logger import CPLog
from bencode import bencode, bdecode
from rtorrent import RTorrent
from scandir import scandir
log = CPLog(__name__)
@@ -238,7 +237,7 @@ class rTorrent(DownloaderBase):
if torrent.is_multi_file() and torrent.directory.endswith(torrent.name):
# Remove empty directories bottom up
try:
for path, _, _ in scandir.walk(sp(torrent.directory), topdown = False):
for path, _, _ in os.walk(sp(torrent.directory), topdown = False):
os.rmdir(path)
except OSError:
log.info('Directory "%s" contains extra files, unable to remove', torrent.directory)
couchpotato/core/plugins/base.py
+1 -2
View File
@@ -16,7 +16,6 @@ from couchpotato.environment import Env
import requests
from requests.packages.urllib3 import Timeout
from requests.packages.urllib3.exceptions import MaxRetryError
from scandir import scandir
from tornado import template
from tornado.web import StaticFileHandler
@@ -144,7 +143,7 @@ class Plugin(object):
def deleteEmptyFolder(self, folder, show_error = True):
folder = sp(folder)
for root, dirs, files in scandir.walk(folder):
for root, dirs, files in os.walk(folder):
for dir_name in dirs:
full_path = os.path.join(root, dir_name)
couchpotato/core/plugins/file.py
+1 -2
View File
@@ -9,7 +9,6 @@ from couchpotato.core.helpers.variable import md5, getExt
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.environment import Env
from scandir import scandir
from tornado.web import StaticFileHandler
@@ -49,7 +48,7 @@ class FileManager(Plugin):
for x in file_dict.keys():
files.extend(file_dict[x])
for f in scandir.scandir(cache_dir):
for f in os.listdir(cache_dir):
if os.path.splitext(f.name)[1] in ['.png', '.jpg', '.jpeg']:
file_path = os.path.join(cache_dir, f.name)
if toUnicode(file_path) not in files:
couchpotato/core/plugins/renamer.py
+7 -8
View File
@@ -14,7 +14,6 @@ from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.environment import Env
from scandir import scandir
from unrar2 import RarFile
import six
from six.moves import filter
@@ -195,7 +194,7 @@ class Renamer(Plugin):
else:
# Get all files from the specified folder
try:
for root, folders, names in scandir.walk(media_folder):
for root, folders, names in os.walk(media_folder):
files.extend([sp(os.path.join(root, name)) for name in names])
except:
log.error('Failed getting files from %s: %s', (media_folder, traceback.format_exc()))
@@ -664,7 +663,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
# Tag all files in release folder
elif release_download['folder']:
for root, folders, names in scandir.walk(sp(release_download['folder'])):
for root, folders, names in os.walk(sp(release_download['folder'])):
tag_files.extend([os.path.join(root, name) for name in names])
for filename in tag_files:
@@ -704,7 +703,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
# Untag all files in release folder
else:
for root, folders, names in scandir.walk(folder):
for root, folders, names in os.walk(folder):
tag_files.extend([sp(os.path.join(root, name)) for name in names if not os.path.splitext(name)[1] == '.ignore'])
if not folder:
@@ -712,7 +711,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
# Find all .ignore files in folder
ignore_files = []
for root, dirnames, filenames in scandir.walk(folder):
for root, dirnames, filenames in os.walk(folder):
ignore_files.extend(fnmatch.filter([sp(os.path.join(root, filename)) for filename in filenames], '*%s.ignore' % tag))
# Match all found ignore files with the tag_files and delete if found
@@ -741,11 +740,11 @@ Remove it if you want it to be renamed (again, or at least let it try again)
# Find tag on all files in release folder
else:
for root, folders, names in scandir.walk(folder):
for root, folders, names in os.walk(folder):
tag_files.extend([sp(os.path.join(root, name)) for name in names if not os.path.splitext(name)[1] == '.ignore'])
# Find all .ignore files in folder
for root, dirnames, filenames in scandir.walk(folder):
for root, dirnames, filenames in os.walk(folder):
ignore_files.extend(fnmatch.filter([sp(os.path.join(root, filename)) for filename in filenames], '*%s.ignore' % tag))
# Match all found ignore files with the tag_files and return True found
@@ -1102,7 +1101,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
check_file_date = False
if not files:
for root, folders, names in scandir.walk(folder):
for root, folders, names in os.walk(folder):
files.extend([sp(os.path.join(root, name)) for name in names])
# Find all archive files
couchpotato/core/plugins/scanner.py
+1 -2
View File
@@ -13,7 +13,6 @@ from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from enzyme.exceptions import NoParserError, ParseError
from guessit import guess_movie_info
from scandir import scandir
from subliminal.videos import Video
import enzyme
from six.moves import filter, map, zip
@@ -150,7 +149,7 @@ class Scanner(Plugin):
check_file_date = True
try:
files = []
for root, dirs, walk_files in scandir.walk(folder, followlinks=True):
for root, dirs, walk_files in os.walk(folder, followlinks=True):
files.extend([sp(os.path.join(sp(root), ss(filename))) for filename in walk_files])
# Break if CP wants to shut down
couchpotato/runner.py
+2 -3
View File
@@ -18,7 +18,6 @@ from couchpotato.api import NonBlockHandler, ApiHandler
from couchpotato.core.event import fireEventAsync, fireEvent
from couchpotato.core.helpers.encoding import sp
from couchpotato.core.helpers.variable import getDataDir, tryInt
from scandir import scandir
from tornado.httpserver import HTTPServer
from tornado.web import Application, StaticFileHandler, RedirectHandler
@@ -99,7 +98,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
existing_backups = []
if not os.path.isdir(backup_path): os.makedirs(backup_path)
for root, dirs, files in scandir.walk(backup_path):
for root, dirs, files in os.walk(backup_path):
for backup_file in files:
ints = re.findall('\d+', backup_file)
@@ -116,7 +115,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
# Create new backup
new_backup = sp(os.path.join(backup_path, '%s.tar.gz' % int(time.time())))
zipf = tarfile.open(new_backup, 'w:gz')
for root, dirs, files in scandir.walk(db_path):
for root, dirs, files in os.walk(db_path):
for zfilename in files:
zipf.add(os.path.join(root, zfilename), arcname = 'database/%s' % os.path.join(root[len(db_path) + 1:], zfilename))
zipf.close()
libs/scandir/.gitattributes
-1
View File
@@ -1 +0,0 @@
* text=auto
libs/scandir/.gitignore
-4
View File
@@ -1,4 +0,0 @@
*.pyc
*.pyd
benchtree
build
libs/scandir/__init__.py
View File
libs/scandir/_scandir.c
-373
View File
@@ -1,373 +0,0 @@
// scandir C speedups
//
// TODO: this is a work in progress!
//
// There's a fair bit of PY_MAJOR_VERSION boilerplate to support both Python 2
// and Python 3 -- the structure of this is taken from here:
// http://docs.python.org/3.3/howto/cporting.html
#include <Python.h>
#include <structseq.h>
#ifdef MS_WINDOWS
#include <windows.h>
#endif
#if PY_MAJOR_VERSION >= 3
#define INITERROR return NULL
#define FROM_LONG PyLong_FromLong
#define FROM_STRING PyUnicode_FromStringAndSize
#else
#define INITERROR return
#define FROM_LONG PyInt_FromLong
#define FROM_STRING PyString_FromStringAndSize
#endif
#ifdef MS_WINDOWS
static PyObject *
win32_error_unicode(char* function, Py_UNICODE* filename)
{
errno = GetLastError();
if (filename)
return PyErr_SetFromWindowsErrWithUnicodeFilename(errno, filename);
else
return PyErr_SetFromWindowsErr(errno);
}
/* Below, we *know* that ugo+r is 0444 */
#if _S_IREAD != 0400
#error Unsupported C library
#endif
static int
attributes_to_mode(DWORD attr)
{
int m = 0;
if (attr & FILE_ATTRIBUTE_DIRECTORY)
m |= _S_IFDIR | 0111; /* IFEXEC for user,group,other */
else
m |= _S_IFREG;
if (attr & FILE_ATTRIBUTE_READONLY)
m |= 0444;
else
m |= 0666;
if (attr & FILE_ATTRIBUTE_REPARSE_POINT)
m |= 0120000; // S_IFLNK
return m;
}
double
filetime_to_time(FILETIME *filetime)
{
const double SECONDS_BETWEEN_EPOCHS = 11644473600.0;
unsigned long long total = (unsigned long long)filetime->dwHighDateTime << 32 |
(unsigned long long)filetime->dwLowDateTime;
return (double)total / 10000000.0 - SECONDS_BETWEEN_EPOCHS;
}
static PyTypeObject StatResultType;
static PyObject *
find_data_to_statresult(WIN32_FIND_DATAW *data)
{
PY_LONG_LONG size;
PyObject *v = PyStructSequence_New(&StatResultType);
if (v == NULL)
return NULL;
size = (PY_LONG_LONG)data->nFileSizeHigh << 32 |
(PY_LONG_LONG)data->nFileSizeLow;
PyStructSequence_SET_ITEM(v, 0, FROM_LONG(attributes_to_mode(data->dwFileAttributes)));
PyStructSequence_SET_ITEM(v, 1, FROM_LONG(0));
PyStructSequence_SET_ITEM(v, 2, FROM_LONG(0));
PyStructSequence_SET_ITEM(v, 3, FROM_LONG(0));
PyStructSequence_SET_ITEM(v, 4, FROM_LONG(0));
PyStructSequence_SET_ITEM(v, 5, FROM_LONG(0));
PyStructSequence_SET_ITEM(v, 6, PyLong_FromLongLong((PY_LONG_LONG)size));
PyStructSequence_SET_ITEM(v, 7, PyFloat_FromDouble(filetime_to_time(&data->ftLastAccessTime)));
PyStructSequence_SET_ITEM(v, 8, PyFloat_FromDouble(filetime_to_time(&data->ftLastWriteTime)));
PyStructSequence_SET_ITEM(v, 9, PyFloat_FromDouble(filetime_to_time(&data->ftCreationTime)));
if (PyErr_Occurred()) {
Py_DECREF(v);
return NULL;
}
return v;
}
static PyStructSequence_Field stat_result_fields[] = {
{"st_mode", "protection bits"},
{"st_ino", "inode"},
{"st_dev", "device"},
{"st_nlink", "number of hard links"},
{"st_uid", "user ID of owner"},
{"st_gid", "group ID of owner"},
{"st_size", "total size, in bytes"},
{"st_atime", "time of last access"},
{"st_mtime", "time of last modification"},
{"st_ctime", "time of last change"},
{0}
};
static PyStructSequence_Desc stat_result_desc = {
"stat_result", /* name */
NULL, /* doc */
stat_result_fields,
10
};
static PyObject *
scandir_helper(PyObject *self, PyObject *args)
{
PyObject *d, *v;
HANDLE hFindFile;
BOOL result;
WIN32_FIND_DATAW wFileData;
Py_UNICODE *wnamebuf;
Py_ssize_t len;
PyObject *po;
PyObject *name_stat;
if (!PyArg_ParseTuple(args, "U:scandir_helper", &po))
return NULL;
/* Overallocate for \\*.*\0 */
len = PyUnicode_GET_SIZE(po);
wnamebuf = malloc((len + 5) * sizeof(wchar_t));
if (!wnamebuf) {
PyErr_NoMemory();
return NULL;
}
wcscpy(wnamebuf, PyUnicode_AS_UNICODE(po));
if (len > 0) {
Py_UNICODE wch = wnamebuf[len-1];
if (wch != L'/' && wch != L'\\' && wch != L':')
wnamebuf[len++] = L'\\';
wcscpy(wnamebuf + len, L"*.*");
}
if ((d = PyList_New(0)) == NULL) {
free(wnamebuf);
return NULL;
}
Py_BEGIN_ALLOW_THREADS
hFindFile = FindFirstFileW(wnamebuf, &wFileData);
Py_END_ALLOW_THREADS
if (hFindFile == INVALID_HANDLE_VALUE) {
int error = GetLastError();
if (error == ERROR_FILE_NOT_FOUND) {
free(wnamebuf);
return d;
}
Py_DECREF(d);
win32_error_unicode("FindFirstFileW", wnamebuf);
free(wnamebuf);
return NULL;
}
do {
/* Skip over . and .. */
if (wcscmp(wFileData.cFileName, L".") != 0 &&
wcscmp(wFileData.cFileName, L"..") != 0) {
v = PyUnicode_FromUnicode(wFileData.cFileName, wcslen(wFileData.cFileName));
if (v == NULL) {
Py_DECREF(d);
d = NULL;
break;
}
name_stat = Py_BuildValue("ON", v, find_data_to_statresult(&wFileData));
if (name_stat == NULL) {
Py_DECREF(v);
Py_DECREF(d);
d = NULL;
break;
}
if (PyList_Append(d, name_stat) != 0) {
Py_DECREF(v);
Py_DECREF(d);
Py_DECREF(name_stat);
d = NULL;
break;
}
Py_DECREF(name_stat);
Py_DECREF(v);
}
Py_BEGIN_ALLOW_THREADS
result = FindNextFileW(hFindFile, &wFileData);
Py_END_ALLOW_THREADS
/* FindNextFile sets error to ERROR_NO_MORE_FILES if
it got to the end of the directory. */
if (!result && GetLastError() != ERROR_NO_MORE_FILES) {
Py_DECREF(d);
win32_error_unicode("FindNextFileW", wnamebuf);
FindClose(hFindFile);
free(wnamebuf);
return NULL;
}
} while (result == TRUE);
if (FindClose(hFindFile) == FALSE) {
Py_DECREF(d);
win32_error_unicode("FindClose", wnamebuf);
free(wnamebuf);
return NULL;
}
free(wnamebuf);
return d;
}
#else // Linux / OS X
#include <dirent.h>
#define NAMLEN(dirent) strlen((dirent)->d_name)
static PyObject *
posix_error_with_allocated_filename(char* name)
{
PyObject *rc = PyErr_SetFromErrnoWithFilename(PyExc_OSError, name);
PyMem_Free(name);
return rc;
}
static PyObject *
scandir_helper(PyObject *self, PyObject *args)
{
char *name = NULL;
PyObject *d, *v, *name_type;
DIR *dirp;
struct dirent *ep;
int arg_is_unicode = 1;
errno = 0;
if (!PyArg_ParseTuple(args, "U:scandir_helper", &v)) {
arg_is_unicode = 0;
PyErr_Clear();
}
if (!PyArg_ParseTuple(args, "et:scandir_helper", Py_FileSystemDefaultEncoding, &name))
return NULL;
Py_BEGIN_ALLOW_THREADS
dirp = opendir(name);
Py_END_ALLOW_THREADS
if (dirp == NULL) {
return posix_error_with_allocated_filename(name);
}
if ((d = PyList_New(0)) == NULL) {
Py_BEGIN_ALLOW_THREADS
closedir(dirp);
Py_END_ALLOW_THREADS
PyMem_Free(name);
return NULL;
}
for (;;) {
errno = 0;
Py_BEGIN_ALLOW_THREADS
ep = readdir(dirp);
Py_END_ALLOW_THREADS
if (ep == NULL) {
if (errno == 0) {
break;
} else {
Py_BEGIN_ALLOW_THREADS
closedir(dirp);
Py_END_ALLOW_THREADS
Py_DECREF(d);
return posix_error_with_allocated_filename(name);
}
}
if (ep->d_name[0] == '.' &&
(NAMLEN(ep) == 1 ||
(ep->d_name[1] == '.' && NAMLEN(ep) == 2)))
continue;
v = FROM_STRING(ep->d_name, NAMLEN(ep));
if (v == NULL) {
Py_DECREF(d);
d = NULL;
break;
}
if (arg_is_unicode) {
PyObject *w;
w = PyUnicode_FromEncodedObject(v,
Py_FileSystemDefaultEncoding,
"strict");
if (w != NULL) {
Py_DECREF(v);
v = w;
}
else {
/* fall back to the original byte string, as
discussed in patch #683592 */
PyErr_Clear();
}
}
name_type = Py_BuildValue("ON", v, FROM_LONG(ep->d_type));
if (name_type == NULL) {
Py_DECREF(v);
Py_DECREF(d);
d = NULL;
break;
}
if (PyList_Append(d, name_type) != 0) {
Py_DECREF(v);
Py_DECREF(d);
Py_DECREF(name_type);
d = NULL;
break;
}
Py_DECREF(name_type);
Py_DECREF(v);
}
Py_BEGIN_ALLOW_THREADS
closedir(dirp);
Py_END_ALLOW_THREADS
PyMem_Free(name);
return d;
}
#endif
static PyMethodDef scandir_methods[] = {
{"scandir_helper", (PyCFunction)scandir_helper, METH_VARARGS, NULL},
{NULL, NULL},
};
#if PY_MAJOR_VERSION >= 3
static struct PyModuleDef moduledef = {
PyModuleDef_HEAD_INIT,
"_scandir",
NULL,
0,
scandir_methods,
NULL,
NULL,
NULL,
NULL,
};
#endif
#if PY_MAJOR_VERSION >= 3
PyObject *
PyInit__scandir(void)
{
PyObject *module = PyModule_Create(&moduledef);
#else
void
init_scandir(void)
{
PyObject *module = Py_InitModule("_scandir", scandir_methods);
#endif
if (module == NULL) {
INITERROR;
}
#ifdef MS_WINDOWS
stat_result_desc.name = "scandir.stat_result";
PyStructSequence_InitType(&StatResultType, &stat_result_desc);
#endif
#if PY_MAJOR_VERSION >= 3
return module;
#endif
}
libs/scandir/scandir.py
-456
View File
@@ -1,456 +0,0 @@
"""scandir, a better directory iterator that exposes all file info OS provides
scandir is a generator version of os.listdir() that returns an iterator over
files in a directory, and also exposes the extra information most OSes provide
while iterating files in a directory.
See README.md or https://github.com/benhoyt/scandir for rationale and docs.
scandir is released under the new BSD 3-clause license. See LICENSE.txt for
the full license text.
"""
from __future__ import division
import ctypes
import os
import stat
import sys
__version__ = '0.3'
__all__ = ['scandir', 'walk']
# Shortcuts to these functions for speed and ease
join = os.path.join
lstat = os.lstat
S_IFDIR = stat.S_IFDIR
S_IFREG = stat.S_IFREG
S_IFLNK = stat.S_IFLNK
# 'unicode' isn't defined on Python 3
try:
unicode
except NameError:
unicode = str
_scandir = None
class GenericDirEntry(object):
__slots__ = ('name', '_lstat', '_path')
def __init__(self, path, name):
self._path = path
self.name = name
self._lstat = None
def lstat(self):
if self._lstat is None:
self._lstat = lstat(join(self._path, self.name))
return self._lstat
def is_dir(self):
try:
self.lstat()
except OSError:
return False
return self._lstat.st_mode & 0o170000 == S_IFDIR
def is_file(self):
try:
self.lstat()
except OSError:
return False
return self._lstat.st_mode & 0o170000 == S_IFREG
def is_symlink(self):
try:
self.lstat()
except OSError:
return False
return self._lstat.st_mode & 0o170000 == S_IFLNK
def __str__(self):
return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name)
__repr__ = __str__
if sys.platform == 'win32':
from ctypes import wintypes
# Various constants from windows.h
INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value
ERROR_FILE_NOT_FOUND = 2
ERROR_NO_MORE_FILES = 18
FILE_ATTRIBUTE_READONLY = 1
FILE_ATTRIBUTE_DIRECTORY = 16
FILE_ATTRIBUTE_REPARSE_POINT = 1024
# Numer of seconds between 1601-01-01 and 1970-01-01
SECONDS_BETWEEN_EPOCHS = 11644473600
kernel32 = ctypes.windll.kernel32
# ctypes wrappers for (wide string versions of) FindFirstFile,
# FindNextFile, and FindClose
FindFirstFile = kernel32.FindFirstFileW
FindFirstFile.argtypes = [
wintypes.LPCWSTR,
ctypes.POINTER(wintypes.WIN32_FIND_DATAW),
]
FindFirstFile.restype = wintypes.HANDLE
FindNextFile = kernel32.FindNextFileW
FindNextFile.argtypes = [
wintypes.HANDLE,
ctypes.POINTER(wintypes.WIN32_FIND_DATAW),
]
FindNextFile.restype = wintypes.BOOL
FindClose = kernel32.FindClose
FindClose.argtypes = [wintypes.HANDLE]
FindClose.restype = wintypes.BOOL
def filetime_to_time(filetime):
"""Convert Win32 FILETIME to time since Unix epoch in seconds."""
total = filetime.dwHighDateTime << 32 | filetime.dwLowDateTime
return total / 10000000 - SECONDS_BETWEEN_EPOCHS
def find_data_to_stat(data):
"""Convert Win32 FIND_DATA struct to stat_result."""
# First convert Win32 dwFileAttributes to st_mode
attributes = data.dwFileAttributes
st_mode = 0
if attributes & FILE_ATTRIBUTE_DIRECTORY:
st_mode |= S_IFDIR | 0o111
else:
st_mode |= S_IFREG
if attributes & FILE_ATTRIBUTE_READONLY:
st_mode |= 0o444
else:
st_mode |= 0o666
if attributes & FILE_ATTRIBUTE_REPARSE_POINT:
st_mode |= S_IFLNK
st_size = data.nFileSizeHigh << 32 | data.nFileSizeLow
st_atime = filetime_to_time(data.ftLastAccessTime)
st_mtime = filetime_to_time(data.ftLastWriteTime)
st_ctime = filetime_to_time(data.ftCreationTime)
# Some fields set to zero per CPython's posixmodule.c: st_ino, st_dev,
# st_nlink, st_uid, st_gid
return os.stat_result((st_mode, 0, 0, 0, 0, 0, st_size, st_atime,
st_mtime, st_ctime))
class Win32DirEntry(object):
__slots__ = ('name', '_lstat', '_find_data')
def __init__(self, name, find_data):
self.name = name
self._lstat = None
self._find_data = find_data
def lstat(self):
if self._lstat is None:
# Lazily convert to stat object, because it's slow, and often
# we only need is_dir() etc
self._lstat = find_data_to_stat(self._find_data)
return self._lstat
def is_dir(self):
return (self._find_data.dwFileAttributes &
FILE_ATTRIBUTE_DIRECTORY != 0)
def is_file(self):
return (self._find_data.dwFileAttributes &
FILE_ATTRIBUTE_DIRECTORY == 0)
def is_symlink(self):
return (self._find_data.dwFileAttributes &
FILE_ATTRIBUTE_REPARSE_POINT != 0)
def __str__(self):
return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name)
__repr__ = __str__
def win_error(error, filename):
exc = WindowsError(error, ctypes.FormatError(error))
exc.filename = filename
return exc
def scandir(path='.', windows_wildcard='*.*'):
"""Like os.listdir(), but yield DirEntry objects instead of returning
a list of names.
"""
# Call FindFirstFile and handle errors
data = wintypes.WIN32_FIND_DATAW()
data_p = ctypes.byref(data)
filename = join(path, windows_wildcard)
handle = FindFirstFile(filename, data_p)
if handle == INVALID_HANDLE_VALUE:
error = ctypes.GetLastError()
if error == ERROR_FILE_NOT_FOUND:
# No files, don't yield anything
return
raise win_error(error, path)
# Call FindNextFile in a loop, stopping when no more files
try:
while True:
# Skip '.' and '..' (current and parent directory), but
# otherwise yield (filename, stat_result) tuple
name = data.cFileName
if name not in ('.', '..'):
yield Win32DirEntry(name, data)
data = wintypes.WIN32_FIND_DATAW()
data_p = ctypes.byref(data)
success = FindNextFile(handle, data_p)
if not success:
error = ctypes.GetLastError()
if error == ERROR_NO_MORE_FILES:
break
raise win_error(error, path)
finally:
if not FindClose(handle):
raise win_error(ctypes.GetLastError(), path)
try:
import _scandir
scandir_helper = _scandir.scandir_helper
class Win32DirEntry(object):
__slots__ = ('name', '_lstat')
def __init__(self, name, lstat):
self.name = name
self._lstat = lstat
def lstat(self):
return self._lstat
def is_dir(self):
return self._lstat.st_mode & 0o170000 == S_IFDIR
def is_file(self):
return self._lstat.st_mode & 0o170000 == S_IFREG
def is_symlink(self):
return self._lstat.st_mode & 0o170000 == S_IFLNK
def __str__(self):
return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name)
__repr__ = __str__
def scandir(path='.'):
for name, stat in scandir_helper(unicode(path)):
yield Win32DirEntry(name, stat)
except ImportError:
pass
# Linux, OS X, and BSD implementation
elif sys.platform.startswith(('linux', 'darwin')) or 'bsd' in sys.platform:
import ctypes.util
DIR_p = ctypes.c_void_p
# Rather annoying how the dirent struct is slightly different on each
# platform. The only fields we care about are d_name and d_type.
class Dirent(ctypes.Structure):
if sys.platform.startswith('linux'):
_fields_ = (
('d_ino', ctypes.c_ulong),
('d_off', ctypes.c_long),
('d_reclen', ctypes.c_ushort),
('d_type', ctypes.c_byte),
('d_name', ctypes.c_char * 256),
)
else:
_fields_ = (
('d_ino', ctypes.c_uint32), # must be uint32, not ulong
('d_reclen', ctypes.c_ushort),
('d_type', ctypes.c_byte),
('d_namlen', ctypes.c_byte),
('d_name', ctypes.c_char * 256),
)
DT_UNKNOWN = 0
DT_DIR = 4
DT_REG = 8
DT_LNK = 10
Dirent_p = ctypes.POINTER(Dirent)
Dirent_pp = ctypes.POINTER(Dirent_p)
libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True)
opendir = libc.opendir
opendir.argtypes = [ctypes.c_char_p]
opendir.restype = DIR_p
readdir_r = libc.readdir_r
readdir_r.argtypes = [DIR_p, Dirent_p, Dirent_pp]
readdir_r.restype = ctypes.c_int
closedir = libc.closedir
closedir.argtypes = [DIR_p]
closedir.restype = ctypes.c_int
file_system_encoding = sys.getfilesystemencoding()
class PosixDirEntry(object):
__slots__ = ('name', '_d_type', '_lstat', '_path')
def __init__(self, path, name, d_type):
self._path = path
self.name = name
self._d_type = d_type
self._lstat = None
def lstat(self):
if self._lstat is None:
self._lstat = lstat(join(self._path, self.name))
return self._lstat
# Ridiculous duplication between these is* functions -- helps a little
# bit with os.walk() performance compared to calling another function.
def is_dir(self):
d_type = self._d_type
if d_type != DT_UNKNOWN:
return d_type == DT_DIR
try:
self.lstat()
except OSError:
return False
return self._lstat.st_mode & 0o170000 == S_IFDIR
def is_file(self):
d_type = self._d_type
if d_type != DT_UNKNOWN:
return d_type == DT_REG
try:
self.lstat()
except OSError:
return False
return self._lstat.st_mode & 0o170000 == S_IFREG
def is_symlink(self):
d_type = self._d_type
if d_type != DT_UNKNOWN:
return d_type == DT_LNK
try:
self.lstat()
except OSError:
return False
return self._lstat.st_mode & 0o170000 == S_IFLNK
def __str__(self):
return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name)
__repr__ = __str__
def posix_error(filename):
errno = ctypes.get_errno()
exc = OSError(errno, os.strerror(errno))
exc.filename = filename
return exc
def scandir(path='.'):
"""Like os.listdir(), but yield DirEntry objects instead of returning
a list of names.
"""
dir_p = opendir(path.encode(file_system_encoding))
if not dir_p:
raise posix_error(path)
try:
result = Dirent_p()
while True:
entry = Dirent()
if readdir_r(dir_p, entry, result):
raise posix_error(path)
if not result:
break
name = entry.d_name.decode(file_system_encoding)
if name not in ('.', '..'):
yield PosixDirEntry(path, name, entry.d_type)
finally:
if closedir(dir_p):
raise posix_error(path)
try:
import _scandir
scandir_helper = _scandir.scandir_helper
def scandir(path='.'):
for name, d_type in scandir_helper(unicode(path)):
yield PosixDirEntry(path, name, d_type)
except ImportError:
pass
# Some other system -- no d_type or stat information
else:
def scandir(path='.'):
"""Like os.listdir(), but yield DirEntry objects instead of returning
a list of names.
"""
for name in os.listdir(path):
yield GenericDirEntry(path, name)
def walk(top, topdown=True, onerror=None, followlinks=False):
"""Like os.walk(), but faster, as it uses scandir() internally."""
# Determine which are files and which are directories
dirs = []
nondirs = []
try:
for entry in scandir(top):
if entry.is_dir():
dirs.append(entry)
else:
nondirs.append(entry)
except OSError as error:
if onerror is not None:
onerror(error)
return
# Yield before recursion if going top down
if topdown:
# Need to do some fancy footwork here as caller is allowed to modify
# dir_names, and we really want them to modify dirs (list of DirEntry
# objects) instead. Keep a mapping of entries keyed by name.
dir_names = []
entries_by_name = {}
for entry in dirs:
dir_names.append(entry.name)
entries_by_name[entry.name] = entry
yield top, dir_names, [e.name for e in nondirs]
dirs = []
for dir_name in dir_names:
entry = entries_by_name.get(dir_name)
if entry is None:
# Only happens when caller creates a new directory and adds it
# to dir_names
entry = GenericDirEntry(top, dir_name)
dirs.append(entry)
# Recurse into sub-directories, following symbolic links if "followlinks"
for entry in dirs:
if followlinks or not entry.is_symlink():
new_path = join(top, entry.name)
for x in walk(new_path, topdown, onerror, followlinks):
yield x
# Yield before recursion if going bottom up
if not topdown:
yield top, [e.name for e in dirs], [e.name for e in nondirs]