From 7357471b9e454e2823c5c4f87c8ad5410f5560d5 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Fri, 11 May 2018 14:01:20 +0200 Subject: [PATCH] Consumer loop: make sleep duration dynamic Make the sleep duration dynamic to account for the time spent in loop_step. This improves responsiveness when repeatedly consuming newly arriving docs. Use float epoch seconds (time.time()) as the time type for MailFetcher.last_checked to allow for natural time arithmetic. --- src/documents/mail.py | 4 ++-- .../management/commands/document_consumer.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/documents/mail.py b/src/documents/mail.py index 1be62527d..d974d57c5 100644 --- a/src/documents/mail.py +++ b/src/documents/mail.py @@ -161,7 +161,7 @@ class MailFetcher(Loggable): self._enabled = bool(self._host) - self.last_checked = datetime.datetime.now() + self.last_checked = time.time() self.consume = consume def pull(self): @@ -188,7 +188,7 @@ class MailFetcher(Loggable): f.write(message.attachment.data) os.utime(file_name, times=(t, t)) - self.last_checked = datetime.datetime.now() + self.last_checked = time.time() def _get_messages(self): diff --git a/src/documents/management/commands/document_consumer.py b/src/documents/management/commands/document_consumer.py index c5fe9baa2..9234617e1 100644 --- a/src/documents/management/commands/document_consumer.py +++ b/src/documents/management/commands/document_consumer.py @@ -59,7 +59,7 @@ class Command(BaseCommand): self.verbosity = options["verbosity"] directory = options["directory"] loop_time = options["loop_time"] - mail_delta = datetime.timedelta(minutes=options["mail_delta"]) + mail_delta = options["mail_delta"] * 60 try: self.file_consumer = Consumer(consume=directory) @@ -83,18 +83,20 @@ class Command(BaseCommand): def loop(self, loop_time, mail_delta): while True: - self.loop_step(mail_delta) - time.sleep(loop_time) + start_time = time.time() if self.verbosity > 1: - print(".", int(time.time())) + print(".", int(start_time)) + self.loop_step(mail_delta, start_time) + # Sleep until the start of the next loop step + time.sleep(max(0, start_time + loop_time - time.time())) - def loop_step(self, mail_delta): + def loop_step(self, mail_delta, time_now=None): # Occasionally fetch mail and store it to be consumed on the next loop # We fetch email when we first start up so that it is not necessary to # wait for 10 minutes after making changes to the config file. next_mail_time = self.mail_fetcher.last_checked + mail_delta - if self.first_iteration or datetime.datetime.now() > next_mail_time: + if self.first_iteration or time_now > next_mail_time: self.first_iteration = False self.mail_fetcher.pull()