From 1cb158a4b582f0543f92e97ed5a074012a090ea2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Sep 2017 04:44:38 +0200 Subject: [PATCH 1/4] move ISO_FORMAT to constants module also: add ISO_FORMAT_NO_USECS --- src/borg/constants.py | 6 ++++++ src/borg/testsuite/archiver.py | 2 -- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/borg/constants.py b/src/borg/constants.py index 6c13a9c98..36d6ee45b 100644 --- a/src/borg/constants.py +++ b/src/borg/constants.py @@ -66,6 +66,12 @@ EXIT_WARNING = 1 # reached normal end of operation, but there were issues EXIT_ERROR = 2 # terminated abruptly, did not reach end of operation +# never use datetime.isoformat(), it is evil. always use one of these: +# datetime.strftime(ISO_FORMAT) # output always includes .microseconds +# datetime.strftime(ISO_FORMAT_NO_USECS) # output never includes microseconds +ISO_FORMAT_NO_USECS = '%Y-%m-%dT%H:%M:%S' +ISO_FORMAT = ISO_FORMAT_NO_USECS + '.%f' + DASHES = '-' * 78 PBKDF2_ITERATIONS = 100000 diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 1bf3ed737..7a5b616a4 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -61,8 +61,6 @@ src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -ISO_FORMAT = '%Y-%m-%dT%H:%M:%S.%f' - def exec_cmd(*args, archiver=None, fork=False, exe=None, input=b'', binary_output=False, **kw): if fork: From eebb117349dff243ea8fd61169678c99b79865dd Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Sep 2017 05:02:44 +0200 Subject: [PATCH 2/4] use ISO_FORMAT* constants --- src/borg/helpers/manifest.py | 2 +- src/borg/helpers/parseformat.py | 2 +- src/borg/helpers/time.py | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/borg/helpers/manifest.py b/src/borg/helpers/manifest.py index e92672f49..632634891 100644 --- a/src/borg/helpers/manifest.py +++ b/src/borg/helpers/manifest.py @@ -166,7 +166,7 @@ def id_str(self): @property def last_timestamp(self): - return datetime.strptime(self.timestamp, "%Y-%m-%dT%H:%M:%S.%f") + return datetime.strptime(self.timestamp, ISO_FORMAT) @classmethod def load(cls, repository, operations, key=None, force_tam_not_required=False): diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index 25114c460..f926bbe9f 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -137,7 +137,7 @@ def __init__(self, dt): def __format__(self, format_spec): if format_spec == '': - format_spec = '%Y-%m-%dT%H:%M:%S' + format_spec = ISO_FORMAT_NO_USECS return self.dt.__format__(format_spec) diff --git a/src/borg/helpers/time.py b/src/borg/helpers/time.py index 2d22794ae..84391bf51 100644 --- a/src/borg/helpers/time.py +++ b/src/borg/helpers/time.py @@ -2,6 +2,8 @@ import time from datetime import datetime, timezone +from ..constants import ISO_FORMAT, ISO_FORMAT_NO_USECS + def to_localtime(ts): """Convert datetime object from UTC to local time zone""" @@ -10,10 +12,8 @@ def to_localtime(ts): def parse_timestamp(timestamp): """Parse a ISO 8601 timestamp string""" - if '.' in timestamp: # microseconds might not be present - return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%f').replace(tzinfo=timezone.utc) - else: - return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc) + fmt = ISO_FORMAT if '.' in timestamp else ISO_FORMAT_NO_USECS + return datetime.strptime(timestamp, fmt).replace(tzinfo=timezone.utc) def timestamp(s): @@ -98,7 +98,7 @@ def isoformat_time(ts: datetime): Format *ts* according to ISO 8601. """ # note: first make all datetime objects tz aware before adding %z here. - return ts.strftime('%Y-%m-%dT%H:%M:%S.%f') + return ts.strftime(ISO_FORMAT) def format_timedelta(td): From 7996a87357e4e67a833bcee5a54ef8b41c491518 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Sep 2017 05:51:00 +0200 Subject: [PATCH 3/4] use safe parse_timestamp to parse timestamps, fixes #2994 also: refactor so it is possible to get tz-unaware datetime objects from parse_timestamp. --- src/borg/helpers/manifest.py | 2 +- src/borg/helpers/time.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/borg/helpers/manifest.py b/src/borg/helpers/manifest.py index 632634891..6d2dceadb 100644 --- a/src/borg/helpers/manifest.py +++ b/src/borg/helpers/manifest.py @@ -166,7 +166,7 @@ def id_str(self): @property def last_timestamp(self): - return datetime.strptime(self.timestamp, ISO_FORMAT) + return parse_timestamp(self.timestamp, tzinfo=None) @classmethod def load(cls, repository, operations, key=None, force_tam_not_required=False): diff --git a/src/borg/helpers/time.py b/src/borg/helpers/time.py index 84391bf51..c5a5ddd81 100644 --- a/src/borg/helpers/time.py +++ b/src/borg/helpers/time.py @@ -10,10 +10,13 @@ def to_localtime(ts): return datetime(*time.localtime((ts - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds())[:6]) -def parse_timestamp(timestamp): +def parse_timestamp(timestamp, tzinfo=timezone.utc): """Parse a ISO 8601 timestamp string""" fmt = ISO_FORMAT if '.' in timestamp else ISO_FORMAT_NO_USECS - return datetime.strptime(timestamp, fmt).replace(tzinfo=timezone.utc) + dt = datetime.strptime(timestamp, fmt) + if tzinfo is not None: + dt = dt.replace(tzinfo=tzinfo) + return dt def timestamp(s): From 928bde8676a1b8cd45c271291af0f15da0308d93 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Sep 2017 06:13:47 +0200 Subject: [PATCH 4/4] get rid of datetime.isoformat to avoid bugs like #2994 --- src/borg/archive.py | 4 ++-- src/borg/helpers/manifest.py | 8 ++++---- src/borg/repository.py | 3 ++- src/borg/testsuite/archiver.py | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 0f4c2d0f6..3cb0b7cfd 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -461,8 +461,8 @@ def save(self, name=None, comment=None, timestamp=None, additional_metadata=None 'cmdline': sys.argv, 'hostname': socket.gethostname(), 'username': getuser(), - 'time': start.isoformat(), - 'time_end': end.isoformat(), + 'time': start.strftime(ISO_FORMAT), + 'time_end': end.strftime(ISO_FORMAT), 'chunker_params': self.chunker_params, } metadata.update(additional_metadata or {}) diff --git a/src/borg/helpers/manifest.py b/src/borg/helpers/manifest.py index 6d2dceadb..eb88c0389 100644 --- a/src/borg/helpers/manifest.py +++ b/src/borg/helpers/manifest.py @@ -64,7 +64,7 @@ def __setitem__(self, name, info): id, ts = info assert isinstance(id, bytes) if isinstance(ts, datetime): - ts = ts.replace(tzinfo=None).isoformat() + ts = ts.replace(tzinfo=None).strftime(ISO_FORMAT) assert isinstance(ts, str) ts = ts.encode() self._archives[name] = {b'id': id, b'time': ts} @@ -236,11 +236,11 @@ def write(self): self.config[b'tam_required'] = True # self.timestamp needs to be strictly monotonically increasing. Clocks often are not set correctly if self.timestamp is None: - self.timestamp = datetime.utcnow().isoformat() + self.timestamp = datetime.utcnow().strftime(ISO_FORMAT) else: prev_ts = self.last_timestamp - incremented = (prev_ts + timedelta(microseconds=1)).isoformat() - self.timestamp = max(incremented, datetime.utcnow().isoformat()) + incremented = (prev_ts + timedelta(microseconds=1)).strftime(ISO_FORMAT) + self.timestamp = max(incremented, datetime.utcnow().strftime(ISO_FORMAT)) # include checks for limits as enforced by limited unpacker (used by load()) assert len(self.archives) <= MAX_ARCHIVES assert all(len(name) <= 255 for name in self.archives) diff --git a/src/borg/repository.py b/src/borg/repository.py index ca55c6988..86b4ae9b7 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -540,7 +540,8 @@ def rename_tmp(file): # Log transaction in append-only mode if self.append_only: with open(os.path.join(self.path, 'transactions'), 'a') as log: - print('transaction %d, UTC time %s' % (transaction_id, datetime.utcnow().isoformat()), file=log) + print('transaction %d, UTC time %s' % ( + transaction_id, datetime.utcnow().strftime(ISO_FORMAT)), file=log) # Write hints file hints_name = 'hints.%d' % transaction_id diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 7a5b616a4..6af6008f3 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -3035,7 +3035,7 @@ def spoof_manifest(self, repository): 'version': 1, 'archives': {}, 'config': {}, - 'timestamp': (datetime.utcnow() + timedelta(days=1)).isoformat(), + 'timestamp': (datetime.utcnow() + timedelta(days=1)).strftime(ISO_FORMAT), }))) repository.commit() @@ -3047,7 +3047,7 @@ def test_fresh_init_tam_required(self): repository.put(Manifest.MANIFEST_ID, key.encrypt(msgpack.packb({ 'version': 1, 'archives': {}, - 'timestamp': (datetime.utcnow() + timedelta(days=1)).isoformat(), + 'timestamp': (datetime.utcnow() + timedelta(days=1)).strftime(ISO_FORMAT), }))) repository.commit()