Merge pull request #22 from gpcimino/feature/copy-if-newer

Add copy_if_newer parameter to functions in copy.py

Author: willmcgugan

.gitignore

@@ -89,3 +89,8 @@ ENV/
 
 # Rope project settings
 .ropeproject
+
+
+#PyCharm
+
+.idea/

fs/copy.py

@@ -13,9 +13,10 @@
 from .path import combine
 from .path import frombase
 from .path import normpath
+from .errors import FSError
 
 
-def copy_fs(src_fs, dst_fs, walker=None):
+def copy_fs(src_fs, dst_fs, walker=None, on_copy=None):
     """
     Copy the contents of one filesystem to another.
 
@@ -29,17 +30,72 @@ def copy_fs(src_fs, dst_fs, walker=None):
         in ``src_fs``. Set this if you only want to consider a sub-set
         of the resources in ``src_fs``.
     :type walker: :class:`~fs.walk.Walker`
+    :param on_copy: A function callback called after a single file copy is executed.
+    :type on_copy: Function, with signature ``(src_fs, src_path, dst_fs, dst_path)``.
+    """
+    return copy_dir(src_fs, '/', dst_fs, '/', walker=walker, on_copy=on_copy)
+
+
+def copy_fs_if_newer(src_fs, dst_fs, walker=None, on_copy=None):
+    """
+    Copy the contents of one filesystem to another. If both source and destination files exist,
+    the copy is executed only if the source file is newer than the destination file.
+    In case modification times of source or destination files are not available,
+    copy file is always executed.
+
+    :param src_fs: Source filesystem.
+    :type src_fs: :type src_fs: FS URL or instance
+    :param src_path: A path to a directory on ``src_fs``.
+    :type src_path: str
+    :param dst_fs: Destination filesystem.
+    :type dst_fs: FS URL or instance
+    :param walker: A walker object that will be used to scan for files
+        in ``src_fs``. Set this if you only want to consider a sub-set
+        of the resources in ``src_fs``.
+    :type walker: :class:`~fs.walk.Walker`
+    :param on_copy: A function callback called after a single file copy is executed.
+    :type on_copy: Function, with signature ``(src_fs, src_path, dst_fs, dst_path)``.
+    """
+    return copy_dir_if_newer(src_fs, '/', dst_fs, '/', walker=walker, on_copy=on_copy)
+
 
+def _source_is_newer(src_fs, src_path, dst_fs, dst_path):
     """
-    copy_dir(src_fs, '/', dst_fs, '/', walker=walker)
+       Determine if source file is newer than destination file.
+
+       :param src_fs: Source filesystem.
+       :type src_fs: :type src_fs: FS URL or instance
+       :param src_path: A path to a directory on ``src_fs``.
+       :type src_path: str
+       :param dst_fs: Destination filesystem.
+       :type dst_fs: FS URL or instance
+       :returns: True if source file is newer than destination file or
+            file modification time cannot be determined. False otherwise.
+       """
+    try:
+        if not dst_fs.exists(dst_path):
+            return True
+        else:
+            namespace = ('details', 'modified')
+
+            src_modified = src_fs.getinfo(src_path, namespace).modified
+            if src_modified is None:
+                return True
+
+            dst_modified = dst_fs.getinfo(dst_path, namespace).modified
+            if dst_modified is None:
+                return True
 
+            return src_modified > dst_modified
+    except FSError:
+        #todo: should log something here
+        #log.error("cannot determine if source file " + src_path + " is newer than destination file " + dst_path + ", thus safely copy the file')
+        return True
 
 def copy_file(src_fs, src_path, dst_fs, dst_path):
     """
     Copy a file from one filesystem to another.
-
-    If the destination exists, and is a file, it will be first
-    truncated.
+    If the destination exists, and is a file, it will be first truncated.
 
     :param src_fs: Source filesystem.
     :type src_fs: FS URL or instance
@@ -49,19 +105,65 @@ def copy_file(src_fs, src_path, dst_fs, dst_path):
     :type dst_fs: FS URL or instance
     :param dst_path: Path to a file on ``dst_fs``.
     :type dst_path: str
+    :returns: True if the file copy was executed, False otherwise.
 
     """
     with manage_fs(src_fs, writeable=False) as src_fs:
         with manage_fs(dst_fs, create=True) as dst_fs:
             if src_fs is dst_fs:
                 # Same filesystem, so we can do a potentially optimized copy
                 src_fs.copy(src_path, dst_path, overwrite=True)
+                return True
             else:
                 # Standard copy
                 with src_fs.lock(), dst_fs.lock():
                     with src_fs.open(src_path, 'rb') as read_file:
                         # There may be an optimized copy available on dst_fs
                         dst_fs.setbinfile(dst_path, read_file)
+                        return True
+
+
+
+def copy_file_if_newer(src_fs, src_path, dst_fs, dst_path):
+    """
+    Copy a file from one filesystem to another.
+    If the destination exists, and is a file, it will be first truncated.
+    If both source and destination files exist,
+    the copy is executed only if the source file is newer than the destination file.
+    In case modification times of source or destination files are not available,
+    copy is always executed.
+
+    :param src_fs: Source filesystem.
+    :type src_fs: FS URL or instance
+    :param src_path: Path to a file on ``src_fs``.
+    :type src_path: str
+    :param dst_fs: Destination filesystem.
+    :type dst_fs: FS URL or instance
+    :param dst_path: Path to a file on ``dst_fs``.
+    :type dst_path: str
+    :returns: True if the file copy was executed, False otherwise.
+
+    """
+    with manage_fs(src_fs, writeable=False) as src_fs:
+        with manage_fs(dst_fs, create=True) as dst_fs:
+            if src_fs is dst_fs:
+                # Same filesystem, so we can do a potentially optimized copy
+                if _source_is_newer(src_fs, src_path, dst_fs, dst_path):
+                    src_fs.copy(src_path, dst_path, overwrite=True)
+                    return True
+                else:
+                    return False
+            else:
+                # Standard copy
+                with src_fs.lock(), dst_fs.lock():
+                    if _source_is_newer(src_fs, src_path, dst_fs, dst_path):
+                        with src_fs.open(src_path, 'rb') as read_file:
+                            # There may be an optimized copy available on dst_fs
+                            dst_fs.setbinfile(dst_path, read_file)
+                            return True
+                    else:
+                        return False
+
 
 
 def copy_structure(src_fs, dst_fs, walker=None):
@@ -86,7 +188,7 @@ def copy_structure(src_fs, dst_fs, walker=None):
                     dst_fs.makedir(dir_path, recreate=True)
 
 
-def copy_dir(src_fs, src_path, dst_fs, dst_path, walker=None):
+def copy_dir(src_fs, src_path, dst_fs, dst_path, walker=None, on_copy=None):
     """
     Copy a directory from one filesystem to another.
 
@@ -101,13 +203,14 @@ def copy_dir(src_fs, src_path, dst_fs, dst_path, walker=None):
         in ``src_fs``. Set this if you only want to consider a sub-set
         of the resources in ``src_fs``.
     :type walker: :class:`~fs.walk.Walker`
+    :param on_copy: A function callback called after a single file copy is executed.
+    :type on_copy: Function, with signature ``(src_fs, src_path, dst_fs, dst_path)``.
 
     """
 
     walker = walker or Walker()
     _src_path = abspath(normpath(src_path))
     _dst_path = abspath(normpath(dst_path))
-
     with manage_fs(src_fs, writeable=False) as src_fs:
         with manage_fs(dst_fs, create=True) as dst_fs:
             with src_fs.lock(), dst_fs.lock():
@@ -123,9 +226,90 @@ def copy_dir(src_fs, src_path, dst_fs, dst_path, walker=None):
                             recreate=True
                         )
                     for info in files:
-                        copy_file(
+                        file_copied = copy_file(
                             src_fs,
                             info.make_path(dir_path),
                             dst_fs,
                             info.make_path(copy_path)
                         )
+                        if file_copied:
+                            if on_copy is not None:
+                                on_copy(src_fs, dir_path, dst_fs, copy_path)
+
+
+def copy_dir_if_newer(src_fs, src_path, dst_fs, dst_path, walker=None, on_copy=None):
+    """
+    Copy a directory from one filesystem to another. If both source and destination files exist,
+    the copy is executed only if the source file is newer than the destination file.
+    In case modification times of source or destination files are not available,
+    copy is always executed.
+
+    :param src_fs: Source filesystem.
+    :type src_fs: FS URL or instance
+    :param src_path: A path to a directory on ``src_fs``.
+    :type src_path: str
+    :param dst_fs: Destination filesystem.
+    :type dst_fs: FS URL or instance
+    :param str dst_path: A path to a directory on ``dst_fs``.
+    :param walker: A walker object that will be used to scan for files
+        in ``src_fs``. Set this if you only want to consider a sub-set
+        of the resources in ``src_fs``.
+    :type walker: :class:`~fs.walk.Walker`
+    :param on_copy: A function callback called after a single file copy is executed.
+    :type on_copy: Function, with signature ``(src_fs, src_path, dst_fs, dst_path)``.
+
+    """
+    walker = walker or Walker()
+    _src_path = abspath(normpath(src_path))
+    _dst_path = abspath(normpath(dst_path))
+    with manage_fs(src_fs, writeable=False) as src_fs:
+        with manage_fs(dst_fs, create=True) as dst_fs:
+            with src_fs.lock(), dst_fs.lock():
+                dst_fs.makedir(_dst_path, recreate=True)
+                namespace = ('details', 'modified')
+                dst_state = {
+                    filepath: info
+                    for filepath, info in walker.info(dst_fs, _dst_path, namespace)
+                    if info.is_file
+                }
+                src_state = [
+                    (filepath, info) for filepath, info in walker.info(src_fs, _src_path, namespace)
+                ]
+                for dir_path, copy_info in src_state:
+                    copy_path = combine(
+                        _dst_path,
+                        frombase(_src_path, dir_path)
+                    )
+                    if copy_info.is_dir:
+                        dst_fs.makedir(copy_path, recreate=True)
+                    if copy_info.is_file:
+                        do_copy = False
+                        if dir_path not in dst_state:
+                            #dst file is not present
+                            do_copy = True
+                        else:
+                            #dst file is present, try to figure out if copy is necessary
+                            src_modified = copy_info.modified
+                            if src_modified is None:
+                                #cannot retrieve src modified timestamp => file will be copied
+                                do_copy = True
+                            dst_modified = dst_state[dir_path].modified
+                            if dst_modified is None:
+                                #cannot retrieve dst modified timestamp => file will be copied
+                                do_copy = True
+                            if src_modified > dst_modified:
+                                #src file is newer than dst file => file will be copied
+                                do_copy = True
+                        if do_copy:
+                            file_copied = copy_file(
+                                src_fs,
+                                dir_path,
+                                dst_fs,
+                                copy_path
+                            )
+                            if file_copied:
+                                if on_copy is not None:
+                                    on_copy(src_fs, dir_path, dst_fs, copy_path)
+
+
+

fs/ftpfs.py

@@ -322,7 +322,7 @@ def __str__(self):
 
     def _open_ftp(self):
         _ftp = FTP()
-        #_ftp.set_debuglevel(2)
+        _ftp.set_debuglevel(0)
         with ftp_errors(self):
             _ftp.connect(self.host, self.port, self.timeout)
             _ftp.login(self.user, self.passwd, self.acct)

fs/osfs.py

@@ -27,7 +27,7 @@
 from .permissions import Permissions
 from .error_tools import convert_os_errors
 from .mode import Mode, validate_open_mode
-
+from .errors import NoURL
 
 log = logging.getLogger('fs.osfs')
 
@@ -305,6 +305,11 @@ def getsyspath(self, path):
         sys_path = self._to_sys_path(path)
         return sys_path
 
+    def geturl(self, path, purpose='download'):
+        if purpose != 'download':
+            raise NoURL(path, purpose)
+        return "file://" + self.getsyspath(path)
+
     def gettype(self, path):
         self.check()
         sys_path = self._to_sys_path(path)