Merge pull request #759 from nachtgeist/public

refactor get_home_dir

borg/helpers.py

@@ -213,19 +213,24 @@ def show_progress(self, item=None, final=False, stream=None, dt=None):
             print(msg, file=stream or sys.stderr, end="\r", flush=True)
 
 
-def get_keys_dir():
-    """Determine where to repository keys and cache"""
-
+def get_home_dir():
+    """Get user's home directory while preferring a possibly set HOME
+    environment variable
+    """
     # os.path.expanduser() behaves differently for '~' and '~someuser' as
     # parameters: when called with an explicit username, the possibly set
     # environment variable HOME is no longer respected. So we have to check if
     # it is set and only expand the user's home directory if HOME is unset.
     if os.environ.get('HOME', ''):
-        home_dir = os.environ.get('HOME')
+        return os.environ.get('HOME')
     else:
-        home_dir = os.path.expanduser('~%s' % os.environ.get('USER'))
+        return os.path.expanduser('~%s' % os.environ.get('USER', ''))
+
+
+def get_keys_dir():
+    """Determine where to repository keys and cache"""
 
-    xdg_config = os.environ.get('XDG_CONFIG_HOME', os.path.join(home_dir, '.config'))
+    xdg_config = os.environ.get('XDG_CONFIG_HOME', os.path.join(get_home_dir(), '.config'))
     keys_dir = os.environ.get('BORG_KEYS_DIR', os.path.join(xdg_config, 'borg', 'keys'))
     if not os.path.exists(keys_dir):
         os.makedirs(keys_dir)
@@ -235,7 +240,7 @@ def get_keys_dir():
 
 def get_cache_dir():
     """Determine where to repository keys and cache"""
-    xdg_cache = os.environ.get('XDG_CACHE_HOME', os.path.join(os.path.expanduser('~'), '.cache'))
+    xdg_cache = os.environ.get('XDG_CACHE_HOME', os.path.join(get_home_dir(), '.cache'))
     cache_dir = os.environ.get('BORG_CACHE_DIR', os.path.join(xdg_cache, 'borg'))
     if not os.path.exists(cache_dir):
         os.makedirs(cache_dir)

borg/remote.py

@@ -10,7 +10,7 @@
 
 from . import __version__
 
-from .helpers import Error, IntegrityError, sysinfo
+from .helpers import Error, IntegrityError, get_home_dir, sysinfo
 from .repository import Repository
 
 import msgpack
@@ -108,8 +108,8 @@ def negotiate(self, versions):
     def open(self, path, create=False, lock_wait=None, lock=True):
         path = os.fsdecode(path)
         if path.startswith('/~'):
-            path = path[1:]
-        path = os.path.realpath(os.path.expanduser(path))
+            path = os.path.join(get_home_dir(), path[2:])
+        path = os.path.realpath(path)
         if self.restrict_to_paths:
             for restrict_to_path in self.restrict_to_paths:
                 if path.startswith(os.path.realpath(restrict_to_path)):

borg/upgrader.py

@@ -6,7 +6,7 @@
 import shutil
 import time
 
-from .helpers import get_keys_dir, get_cache_dir, ProgressIndicatorPercent
+from .helpers import get_home_dir, get_keys_dir, get_cache_dir, ProgressIndicatorPercent
 from .locking import UpgradableLock
 from .repository import Repository, MAGIC
 from .key import KeyfileKey, KeyfileNotFoundError
@@ -187,7 +187,7 @@ def convert_cache(self, dryrun):
         """
         # copy of attic's get_cache_dir()
         attic_cache_dir = os.environ.get('ATTIC_CACHE_DIR',
-                                         os.path.join(os.path.expanduser('~'),
+                                         os.path.join(get_home_dir(),
                                                       '.cache', 'attic'))
         attic_cache_dir = os.path.join(attic_cache_dir, hexlify(self.id).decode('ascii'))
         borg_cache_dir = os.path.join(get_cache_dir(), hexlify(self.id).decode('ascii'))
@@ -248,7 +248,7 @@ class AtticKeyfileKey(KeyfileKey):
     def get_keys_dir():
         """Determine where to repository keys and cache"""
         return os.environ.get('ATTIC_KEYS_DIR',
-                              os.path.join(os.path.expanduser('~'), '.attic', 'keys'))
+                              os.path.join(get_home_dir(), '.attic', 'keys'))
 
     @classmethod
     def find_key_file(cls, repository):
@@ -308,7 +308,7 @@ class Borg0xxKeyfileKey(KeyfileKey):
     @staticmethod
     def get_keys_dir():
         return os.environ.get('BORG_KEYS_DIR',
-                              os.path.join(os.path.expanduser('~'), '.borg', 'keys'))
+                              os.path.join(get_home_dir(), '.borg', 'keys'))
 
     @classmethod
     def find_key_file(cls, repository):

docs/api.rst

@@ -2,75 +2,75 @@
 API Documentation
 =================
 
-.. automodule:: borg.archiver
+.. automodule:: borg.xattr
     :members:
     :undoc-members:
 
-.. automodule:: borg.upgrader
+.. automodule:: borg.hash_sizes
     :members:
     :undoc-members:
 
 .. automodule:: borg.archive
     :members:
     :undoc-members:
 
-.. automodule:: borg.fuse
+.. automodule:: borg.repository
     :members:
     :undoc-members:
 
 .. automodule:: borg.platform
     :members:
     :undoc-members:
 
-.. automodule:: borg.locking
+.. automodule:: borg.cache
     :members:
     :undoc-members:
 
-.. automodule:: borg.shellpattern
+.. automodule:: borg.helpers
     :members:
     :undoc-members:
 
-.. automodule:: borg.repository
+.. automodule:: borg.lrucache
     :members:
     :undoc-members:
 
-.. automodule:: borg.lrucache
+.. automodule:: borg.key
     :members:
     :undoc-members:
 
-.. automodule:: borg.remote
+.. automodule:: borg.upgrader
     :members:
     :undoc-members:
 
-.. automodule:: borg.hash_sizes
+.. automodule:: borg.shellpattern
     :members:
     :undoc-members:
 
-.. automodule:: borg.xattr
+.. automodule:: borg.locking
     :members:
     :undoc-members:
 
-.. automodule:: borg.helpers
+.. automodule:: borg.fuse
     :members:
     :undoc-members:
 
-.. automodule:: borg.cache
+.. automodule:: borg.logger
     :members:
     :undoc-members:
 
-.. automodule:: borg.key
+.. automodule:: borg.archiver
     :members:
     :undoc-members:
 
-.. automodule:: borg.logger
+.. automodule:: borg.remote
     :members:
     :undoc-members:
 
-.. automodule:: borg.platform_darwin
+.. automodule:: borg.chunker
     :members:
     :undoc-members:
 
-.. automodule:: borg.platform_linux
+.. automodule:: borg.platform_freebsd
     :members:
     :undoc-members:
 
@@ -82,14 +82,14 @@ API Documentation
     :members:
     :undoc-members:
 
-.. automodule:: borg.chunker
+.. automodule:: borg.platform_linux
     :members:
     :undoc-members:
 
 .. automodule:: borg.crypto
     :members:
     :undoc-members:
 
-.. automodule:: borg.platform_freebsd
+.. automodule:: borg.platform_darwin
     :members:
     :undoc-members:

docs/usage/break-lock.rst.inc

@@ -4,8 +4,9 @@ borg break-lock
 ---------------
 ::
 
-    usage: borg break-lock [-h] [-v] [--debug] [--lock-wait N] [--show-rc]
-                           [--no-files-cache] [--umask M] [--remote-path PATH]
+    usage: borg break-lock [-h] [-v] [--debug] [--lock-wait N] [--show-version]
+                           [--show-rc] [--no-files-cache] [--umask M]
+                           [--remote-path PATH]
                            REPOSITORY
 
     Break the repository lock (e.g. in case it was left by a dead borg.
@@ -20,6 +21,7 @@ borg break-lock
                             INFO
       --debug               enable debug output, work on log level DEBUG
       --lock-wait N         wait for the lock, but max. N seconds (default: 1).
+      --show-version        show/log the borg version
       --show-rc             show/log the return code (rc)
       --no-files-cache      do not load/update the file metadata cache used to
                             detect unchanged files

docs/usage/change-passphrase.rst.inc

@@ -4,9 +4,9 @@ borg change-passphrase
 ----------------------
 ::
 
-    usage: borg change-passphrase [-h] [-v] [--debug] [--lock-wait N] [--show-rc]
-                                  [--no-files-cache] [--umask M]
-                                  [--remote-path PATH]
+    usage: borg change-passphrase [-h] [-v] [--debug] [--lock-wait N]
+                                  [--show-version] [--show-rc] [--no-files-cache]
+                                  [--umask M] [--remote-path PATH]
                                   [REPOSITORY]
 
     Change repository key file passphrase
@@ -21,6 +21,7 @@ borg change-passphrase
                             INFO
       --debug               enable debug output, work on log level DEBUG
       --lock-wait N         wait for the lock, but max. N seconds (default: 1).
+      --show-version        show/log the borg version
       --show-rc             show/log the return code (rc)
       --no-files-cache      do not load/update the file metadata cache used to
                             detect unchanged files

docs/usage/check.rst.inc

@@ -4,10 +4,10 @@ borg check
 ----------
 ::
 
-    usage: borg check [-h] [-v] [--debug] [--lock-wait N] [--show-rc]
-                      [--no-files-cache] [--umask M] [--remote-path PATH]
-                      [--repository-only] [--archives-only] [--repair]
-                      [--save-space] [--last N] [-P PREFIX]
+    usage: borg check [-h] [-v] [--debug] [--lock-wait N] [--show-version]
+                      [--show-rc] [--no-files-cache] [--umask M]
+                      [--remote-path PATH] [--repository-only] [--archives-only]
+                      [--repair] [--save-space] [--last N] [-P PREFIX]
                       [REPOSITORY_OR_ARCHIVE]
 
     Check repository consistency
@@ -23,6 +23,7 @@ borg check
                             INFO
       --debug               enable debug output, work on log level DEBUG
       --lock-wait N         wait for the lock, but max. N seconds (default: 1).
+      --show-version        show/log the borg version
       --show-rc             show/log the return code (rc)
       --no-files-cache      do not load/update the file metadata cache used to
                             detect unchanged files

docs/usage/create.rst.inc

@@ -4,15 +4,16 @@ borg create
 -----------
 ::
 
-    usage: borg create [-h] [-v] [--debug] [--lock-wait N] [--show-rc]
-                       [--no-files-cache] [--umask M] [--remote-path PATH] [-s]
-                       [-p] [--list] [--filter STATUSCHARS] [-e PATTERN]
+    usage: borg create [-h] [-v] [--debug] [--lock-wait N] [--show-version]
+                       [--show-rc] [--no-files-cache] [--umask M]
+                       [--remote-path PATH] [-s] [-p] [--list]
+                       [--filter STATUSCHARS] [-e PATTERN]
                        [--exclude-from EXCLUDEFILE] [--exclude-caches]
                        [--exclude-if-present FILENAME] [--keep-tag-files]
                        [-c SECONDS] [-x] [--numeric-owner]
                        [--timestamp yyyy-mm-ddThh:mm:ss]
                        [--chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE]
-                       [-C COMPRESSION] [--read-special] [-n]
+                       [--ignore-inode] [-C COMPRESSION] [--read-special] [-n]
                        ARCHIVE PATH [PATH ...]
 
     Create new archive
@@ -29,6 +30,7 @@ borg create
                             INFO
       --debug               enable debug output, work on log level DEBUG
       --lock-wait N         wait for the lock, but max. N seconds (default: 1).
+      --show-version        show/log the borg version
       --show-rc             show/log the return code (rc)
       --no-files-cache      do not load/update the file metadata cache used to
                             detect unchanged files
@@ -60,6 +62,8 @@ borg create
                             alternatively, give a reference file/directory.
       --chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE
                             specify the chunker parameters. default: 19,23,21,4095
+      --ignore-inode        ignore inode data in the file metadata cache used to
+                            detect unchanged files.
       -C COMPRESSION, --compression COMPRESSION
                             select compression algorithm (and level): none == no
                             compression (default), lz4 == lz4, zlib == zlib
@@ -77,4 +81,10 @@ This command creates a backup archive containing all files found while recursive
 traversing all paths specified. The archive will consume almost no disk space for
 files or parts of files that have already been stored in other archives.
 
+
+To speed up pulling backups over sshfs and similar network file systems which do
+not provide correct inode information the --ignore-inode flag can be used. This
+potentially decreases reliability of change detection, while avoiding always reading
+all files on these file systems.
+
 See the output of the "borg help patterns" command for more help on exclude patterns.

docs/usage/debug-delete-obj.rst.inc

@@ -4,9 +4,9 @@ borg debug-delete-obj
 ---------------------
 ::
 
-    usage: borg debug-delete-obj [-h] [-v] [--debug] [--lock-wait N] [--show-rc]
-                                 [--no-files-cache] [--umask M]
-                                 [--remote-path PATH]
+    usage: borg debug-delete-obj [-h] [-v] [--debug] [--lock-wait N]
+                                 [--show-version] [--show-rc] [--no-files-cache]
+                                 [--umask M] [--remote-path PATH]
                                  [REPOSITORY] IDs [IDs ...]
 
     delete the objects with the given IDs from the repo
@@ -22,6 +22,7 @@ borg debug-delete-obj
                             INFO
       --debug               enable debug output, work on log level DEBUG
       --lock-wait N         wait for the lock, but max. N seconds (default: 1).
+      --show-version        show/log the borg version
       --show-rc             show/log the return code (rc)
       --no-files-cache      do not load/update the file metadata cache used to
                             detect unchanged files

docs/usage/debug-dump-archive-items.rst.inc

@@ -5,8 +5,9 @@ borg debug-dump-archive-items
 ::
 
     usage: borg debug-dump-archive-items [-h] [-v] [--debug] [--lock-wait N]
-                                         [--show-rc] [--no-files-cache]
-                                         [--umask M] [--remote-path PATH]
+                                         [--show-version] [--show-rc]
+                                         [--no-files-cache] [--umask M]
+                                         [--remote-path PATH]
                                          ARCHIVE
 
     dump (decrypted, decompressed) archive items metadata (not: data)
@@ -21,6 +22,7 @@ borg debug-dump-archive-items
                             INFO
       --debug               enable debug output, work on log level DEBUG
       --lock-wait N         wait for the lock, but max. N seconds (default: 1).
+      --show-version        show/log the borg version
       --show-rc             show/log the return code (rc)
       --no-files-cache      do not load/update the file metadata cache used to
                             detect unchanged files

docs/usage/debug-get-obj.rst.inc

@@ -4,8 +4,9 @@ borg debug-get-obj
 ------------------
 ::
 
-    usage: borg debug-get-obj [-h] [-v] [--debug] [--lock-wait N] [--show-rc]
-                              [--no-files-cache] [--umask M] [--remote-path PATH]
+    usage: borg debug-get-obj [-h] [-v] [--debug] [--lock-wait N] [--show-version]
+                              [--show-rc] [--no-files-cache] [--umask M]
+                              [--remote-path PATH]
                               [REPOSITORY] ID PATH
 
     get object contents from the repository and write it into file
@@ -22,6 +23,7 @@ borg debug-get-obj
                             INFO
       --debug               enable debug output, work on log level DEBUG
       --lock-wait N         wait for the lock, but max. N seconds (default: 1).
+      --show-version        show/log the borg version
       --show-rc             show/log the return code (rc)
       --no-files-cache      do not load/update the file metadata cache used to
                             detect unchanged files

docs/usage/debug-put-obj.rst.inc

@@ -4,8 +4,9 @@ borg debug-put-obj
 ------------------
 ::
 
-    usage: borg debug-put-obj [-h] [-v] [--debug] [--lock-wait N] [--show-rc]
-                              [--no-files-cache] [--umask M] [--remote-path PATH]
+    usage: borg debug-put-obj [-h] [-v] [--debug] [--lock-wait N] [--show-version]
+                              [--show-rc] [--no-files-cache] [--umask M]
+                              [--remote-path PATH]
                               [REPOSITORY] PATH [PATH ...]
 
     put file(s) contents into the repository
@@ -21,6 +22,7 @@ borg debug-put-obj
                             INFO
       --debug               enable debug output, work on log level DEBUG
       --lock-wait N         wait for the lock, but max. N seconds (default: 1).
+      --show-version        show/log the borg version
       --show-rc             show/log the return code (rc)
       --no-files-cache      do not load/update the file metadata cache used to
                             detect unchanged files