Merge branch 'fix-history-fetch' into 'master'

Fix many MAM issues

Closes #3516, #3496, #3498, #3506, #3522, and #3493

See merge request poezio/poezio!105

12 files changed, 547 insertions, 155 deletions

diff --git a/poezio/common.py b/poezio/common.py
index 7cddc306..98870dda 100644
--- a/poezio/common.py
+++ b/poezio/common.py
@@ -8,7 +8,11 @@
 Various useful functions.
 """
 
-from datetime import datetime, timedelta
+from datetime import (
+    datetime,
+    timedelta,
+    timezone,
+)
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union
 
@@ -488,3 +492,14 @@ def unique_prefix_of(a: str, b: str) -> str:
             return a[:i+1]
     # both are equal, return a
     return a
+
+
+def to_utc(time: datetime) -> datetime:
+    """Convert a datetime-aware time zone into raw UTC"""
+    tzone = datetime.now().astimezone().tzinfo
+    if time.tzinfo is not None:  # Convert to UTC
+        time = time.astimezone(tz=timezone.utc)
+    else:  # Assume local tz, convert to URC
+        time = time.replace(tzinfo=tzone).astimezone(tz=timezone.utc)
+    # Return an offset-naive datetime
+    return time.replace(tzinfo=None)
diff --git a/poezio/core/core.py b/poezio/core/core.py
index 8ac88dd4..973c9103 100644
--- a/poezio/core/core.py
+++ b/poezio/core/core.py
@@ -2075,16 +2075,7 @@ class Core:
             # do not join rooms that do not have autojoin
             # but display them anyway
             if bm.autojoin:
-                muc.join_groupchat(
-                    self,
-                    bm.jid,
-                    nick,
-                    passwd=bm.password,
-                    status=self.status.message,
-                    show=self.status.show,
-                    tab=tab)
-                if tab._text_buffer.last_message is None:
-                    asyncio.ensure_future(mam.on_tab_open(tab))
+                tab.join()
 
     def check_bookmark_storage(self, features):
         private = 'jabber:iq:private' in features
diff --git a/poezio/mam.py b/poezio/mam.py
index 50dad4a3..371b34dd 100644
--- a/poezio/mam.py
+++ b/poezio/mam.py
@@ -6,34 +6,49 @@
     XEP-0313: Message Archive Management(MAM).
 """
 
+import asyncio
+import logging
 import random
 from datetime import datetime, timedelta, timezone
 from hashlib import md5
-from typing import Optional, Callable
+from typing import (
+    Any,
+    AsyncIterable,
+    Callable,
+    Dict,
+    List,
+    Optional,
+)
 
-from slixmpp import JID
+from slixmpp import JID, Message as SMessage
 from slixmpp.exceptions import IqError, IqTimeout
 from poezio.theming import get_theme
 from poezio import tabs
 from poezio import xhtml, colors
 from poezio.config import config
-from poezio.text_buffer import TextBuffer
-from poezio.ui.types import Message
+from poezio.common import to_utc
+from poezio.text_buffer import TextBuffer, HistoryGap
+from poezio.ui.types import (
+    BaseMessage,
+    EndOfArchive,
+    Message,
+)
 
 
+log = logging.getLogger(__name__)
+
 class DiscoInfoException(Exception): pass
 class MAMQueryException(Exception): pass
 class NoMAMSupportException(Exception): pass
 
 
-def add_line(
-        tab,
-        text_buffer: TextBuffer,
+def make_line(
+        tab: tabs.ChatTab,
         text: str,
         time: datetime,
         nick: str,
-        top: bool,
-    ) -> None:
+        identifier: str = '',
+    ) -> Message:
     """Adds a textual entry in the TextBuffer"""
 
     # Convert to local timezone
@@ -61,150 +76,188 @@ def add_line(
             color = xhtml.colors.get(color)
             color = (color, -1)
     else:
-        nick = nick.split('/')[0]
-        color = get_theme().COLOR_OWN_NICK
-    text_buffer.add_message(
-        Message(
-            txt=text,
-            time=time,
-            nickname=nick,
-            nick_color=color,
-            history=True,
-            user=None,
-            top=top,
-        )
+        if nick.split('/')[0] == tab.core.xmpp.boundjid.bare:
+            color = get_theme().COLOR_OWN_NICK
+        else:
+            color = get_theme().COLOR_REMOTE_USER
+        nick = tab.get_nick()
+    return Message(
+        txt=text,
+        identifier=identifier,
+        time=time,
+        nickname=nick,
+        nick_color=color,
+        history=True,
+        user=None,
     )
 
-
-async def query(
+async def get_mam_iterator(
         core,
         groupchat: bool,
         remote_jid: JID,
         amount: int,
-        reverse: bool,
-        start: Optional[datetime] = None,
-        end: Optional[datetime] = None,
+        reverse: bool = True,
+        start: Optional[str] = None,
+        end: Optional[str] = None,
         before: Optional[str] = None,
-        callback: Optional[Callable] = None,
-    ) -> None:
+    ) -> AsyncIterable[Message]:
+    """Get an async iterator for this mam query"""
     try:
         query_jid = remote_jid if groupchat else JID(core.xmpp.boundjid.bare)
         iq = await core.xmpp.plugin['xep_0030'].get_info(jid=query_jid)
     except (IqError, IqTimeout):
-        raise DiscoInfoException
+        raise DiscoInfoException()
     if 'urn:xmpp:mam:2' not in iq['disco_info'].get_features():
-        raise NoMAMSupportException
+        raise NoMAMSupportException()
 
     args = {
         'iterator': True,
         'reverse': reverse,
-    }
+    }  # type: Dict[str, Any]
 
     if groupchat:
         args['jid'] = remote_jid
     else:
         args['with_jid'] = remote_jid
 
-    args['rsm'] = {'max': amount}
-    if reverse:
-        if before is not None:
-            args['rsm']['before'] = before
-        else:
-            args['end'] = end
-    else:
-        args['rsm']['start'] = start
-        if before is not None:
-            args['rsm']['end'] = end
-    try:
-        results = core.xmpp['xep_0313'].retrieve(**args)
-    except (IqError, IqTimeout):
-        raise MAMQueryException
-    if callback is not None:
-        callback(results)
+    if amount > 0:
+        args['rsm'] = {'max': amount}
+    args['start'] = start
+    args['end'] = end
+    return core.xmpp['xep_0313'].retrieve(**args)
 
-    return results
 
+def _parse_message(msg: SMessage) -> Dict:
+    """Parse info inside a MAM forwarded message"""
+    forwarded = msg['mam_result']['forwarded']
+    message = forwarded['stanza']
+    return {
+        'time': forwarded['delay']['stamp'],
+        'nick': str(message['from']),
+        'text': message['body'],
+        'identifier': message['origin-id']
+    }
 
-async def add_messages_to_buffer(tab, top: bool, results, amount: int) -> bool:
-    """Prepends or appends messages to the tab text_buffer"""
 
+async def retrieve_messages(tab: tabs.ChatTab,
+                            results: AsyncIterable[SMessage],
+                            amount: int = 100) -> List[BaseMessage]:
+    """Run the MAM query and put messages in order"""
     text_buffer = tab._text_buffer
     msg_count = 0
     msgs = []
-    async for rsm in results:
-        if top:
+    to_add = []
+    try:
+        async for rsm in results:
             for msg in rsm['mam']['results']:
                 if msg['mam_result']['forwarded']['stanza'] \
-                .xml.find('{%s}%s' % ('jabber:client', 'body')) is not None:
-                    msgs.append(msg)
-                if msg_count == amount:
-                    tab.core.refresh_window()
-                    return False
+                        .xml.find('{%s}%s' % ('jabber:client', 'body')) is not None:
+                    args = _parse_message(msg)
+                    msgs.append(make_line(tab, **args))
+            for msg in reversed(msgs):
+                to_add.append(msg)
                 msg_count += 1
-            msgs.reverse()
-            for msg in msgs:
-                forwarded = msg['mam_result']['forwarded']
-                timestamp = forwarded['delay']['stamp']
-                message = forwarded['stanza']
-                tab.last_stanza_id = msg['mam_result']['id']
-                nick = str(message['from'])
-                add_line(tab, text_buffer, message['body'], timestamp, nick, top)
-        else:
-            for msg in rsm['mam']['results']:
-                forwarded = msg['mam_result']['forwarded']
-                timestamp = forwarded['delay']['stamp']
-                message = forwarded['stanza']
-                nick = str(message['from'])
-                add_line(tab, text_buffer, message['body'], timestamp, nick, top)
-                tab.core.refresh_window()
-    return False
+                if msg_count == amount:
+                    to_add.reverse()
+                    return to_add
+            msgs = []
+        to_add.reverse()
+        return to_add
+    except (IqError, IqTimeout) as exc:
+        log.debug('Unable to complete MAM query: %s', exc, exc_info=True)
+        raise MAMQueryException('Query interrupted')
 
 
-async def fetch_history(tab, end: Optional[datetime] = None, amount: Optional[int] = None):
+async def fetch_history(tab: tabs.ChatTab,
+                        start: Optional[datetime] = None,
+                        end: Optional[datetime] = None,
+                        amount: int = 100) -> List[BaseMessage]:
     remote_jid = tab.jid
-    before = tab.last_stanza_id
+    if not end:
+        for msg in tab._text_buffer.messages:
+            if isinstance(msg, Message):
+                end = msg.time
+                end -= timedelta(microseconds=1)
+                break
     if end is None:
         end = datetime.now()
-    tzone = datetime.now().astimezone().tzinfo
-    end = end.replace(tzinfo=tzone).astimezone(tz=timezone.utc)
-    end = end.replace(tzinfo=None)
-    end = datetime.strftime(end, '%Y-%m-%dT%H:%M:%SZ')
-
-    if amount >= 100:
-        amount = 99
+    end = to_utc(end)
+    end_str = datetime.strftime(end, '%Y-%m-%dT%H:%M:%SZ')
 
-    groupchat = isinstance(tab, tabs.MucTab)
+    start_str = None
+    if start is not None:
+        start = to_utc(start)
+        start_str = datetime.strftime(start, '%Y-%m-%dT%H:%M:%SZ')
 
-    results = await query(
-        tab.core,
-        groupchat,
-        remote_jid,
-        amount,
+    mam_iterator = await get_mam_iterator(
+        core=tab.core,
+        groupchat=isinstance(tab, tabs.MucTab),
+        remote_jid=remote_jid,
+        amount=amount,
+        end=end_str,
+        start=start_str,
         reverse=True,
-        end=end,
-        before=before,
     )
-    query_status = await add_messages_to_buffer(tab, True, results, amount)
-    tab.query_status = query_status
+    return await retrieve_messages(tab, mam_iterator, amount)
 
+async def fill_missing_history(tab: tabs.ChatTab, gap: HistoryGap) -> None:
+    start = gap.last_timestamp_before_leave
+    end = gap.first_timestamp_after_join
+    if start:
+        start = start + timedelta(seconds=1)
+    if end:
+        end = end - timedelta(seconds=1)
+    try:
+        messages = await fetch_history(tab, start=start, end=end, amount=999)
+        tab._text_buffer.add_history_messages(messages, gap=gap)
+        if messages:
+            tab.core.refresh_window()
+    except (NoMAMSupportException, MAMQueryException, DiscoInfoException):
+        return
+    finally:
+        tab.query_status = False
 
-async def on_tab_open(tab) -> None:
+async def on_new_tab_open(tab: tabs.ChatTab) -> None:
+    """Called when opening a new tab"""
     amount = 2 * tab.text_win.height
     end = datetime.now()
-    tab.query_status = True
     for message in tab._text_buffer.messages:
-        time = message.time
-        if time < end:
-            end = time
-    end = end + timedelta(seconds=-1)
+        if isinstance(message, Message) and to_utc(message.time) < to_utc(end):
+            end = message.time
+            break
+    end = end - timedelta(microseconds=1)
     try:
-        await fetch_history(tab, end=end, amount=amount)
+        messages = await fetch_history(tab, end=end, amount=amount)
+        tab._text_buffer.add_history_messages(messages)
+        if messages:
+            tab.core.refresh_window()
     except (NoMAMSupportException, MAMQueryException, DiscoInfoException):
-        tab.query_status = False
         return None
+    finally:
+        tab.query_status = False
+
+
+def schedule_tab_open(tab: tabs.ChatTab) -> None:
+    """Set the query status and schedule a MAM query"""
+    tab.query_status = True
+    asyncio.ensure_future(on_tab_open(tab))
+
 
+async def on_tab_open(tab: tabs.ChatTab) -> None:
+    gap = tab._text_buffer.find_last_gap_muc()
+    if gap is None or not gap.leave_message:
+        await on_new_tab_open(tab)
+    else:
+        await fill_missing_history(tab, gap)
+
+
+def schedule_scroll_up(tab: tabs.ChatTab) -> None:
+    """Set query status and schedule a scroll up"""
+    tab.query_status = True
+    asyncio.ensure_future(on_scroll_up(tab))
 
-async def on_scroll_up(tab) -> None:
+
+async def on_scroll_up(tab: tabs.ChatTab) -> None:
     tw = tab.text_win
 
     # If position in the tab is < two screen pages, then fetch MAM, so that we
@@ -212,22 +265,31 @@ async def on_scroll_up(tab) -> None:
     # join if not already available.
     total, pos, height = len(tw.built_lines), tw.pos, tw.height
     rest = (total - pos) // height
-    # Not resetting the state of query_status here, it is changed only after the
-    # query is complete (in fetch_history)
-    # This is done to stop message repetition, eg: if the user presses PageUp continuously.
-    tab.query_status = True
 
     if rest > 1:
+        tab.query_status = False
         return None
 
     try:
         # XXX: Do we want to fetch a possibly variable number of messages?
         # (InfoTab changes height depending on the type of messages, see
         # `information_buffer_popup_on`).
-        await fetch_history(tab, amount=height)
+        messages = await fetch_history(tab, amount=height)
+        last_message_exists = False
+        if tab._text_buffer.messages:
+            last_message = tab._text_buffer.messages[0]
+            last_message_exists = True
+        if not messages and last_message_exists and not isinstance(last_message, EndOfArchive):
+            time = tab._text_buffer.messages[0].time
+            messages = [EndOfArchive('End of archive reached', time=time)]
+        tab._text_buffer.add_history_messages(messages)
+        if messages:
+            tab.core.refresh_window()
     except NoMAMSupportException:
         tab.core.information('MAM not supported for %r' % tab.jid, 'Info')
         return None
     except (MAMQueryException, DiscoInfoException):
         tab.core.information('An error occured when fetching MAM for %r' % tab.jid, 'Error')
         return None
+    finally:
+        tab.query_status = False
diff --git a/poezio/tabs/basetabs.py b/poezio/tabs/basetabs.py
index fbb0c4cf..a42ee41b 100644
--- a/poezio/tabs/basetabs.py
+++ b/poezio/tabs/basetabs.py
@@ -32,7 +32,6 @@ from typing import (
 )
 
 from poezio import (
-    mam,
     poopt,
     timed_events,
     xhtml,
@@ -493,12 +492,11 @@ class ChatTab(Tab):
         self._jid = jid
         #: Is the tab currently requesting MAM data?
         self.query_status = False
-        self.last_stanza_id = None
-
         self._name = jid.full  # type: Optional[str]
-        self.text_win = None
+        self.text_win = windows.TextWin()
         self.directed_presence = None
         self._text_buffer = TextBuffer()
+        self._text_buffer.add_window(self.text_win)
         self.chatstate = None  # can be "active", "composing", "paused", "gone", "inactive"
         # We keep a reference of the event that will set our chatstate to "paused", so that
         # we can delete it or change it if we need to
@@ -926,7 +924,8 @@ class ChatTab(Tab):
 
     def on_scroll_up(self):
         if not self.query_status:
-            asyncio.ensure_future(mam.on_scroll_up(tab=self))
+            from poezio import mam
+            mam.schedule_scroll_up(tab=self)
         return self.text_win.scroll_up(self.text_win.height - 1)
 
     def on_scroll_down(self):
diff --git a/poezio/tabs/conversationtab.py b/poezio/tabs/conversationtab.py
index 70005f0f..5950e4cb 100644
--- a/poezio/tabs/conversationtab.py
+++ b/poezio/tabs/conversationtab.py
@@ -48,8 +48,6 @@ class ConversationTab(OneToOneTab):
         self.nick = None
         self.nick_sent = False
         self.state = 'normal'
-        self.text_win = windows.TextWin()
-        self._text_buffer.add_window(self.text_win)
         self.upper_bar = windows.ConversationStatusMessageWin()
         self.input = windows.MessageInput()
         # keys
diff --git a/poezio/tabs/muctab.py b/poezio/tabs/muctab.py
index d16ac58a..751509a7 100644
--- a/poezio/tabs/muctab.py
+++ b/poezio/tabs/muctab.py
@@ -31,7 +31,7 @@ from poezio import multiuserchat as muc
 from poezio import timed_events
 from poezio import windows
 from poezio import xhtml
-from poezio.common import safeJID
+from poezio.common import safeJID, to_utc
 from poezio.config import config
 from poezio.core.structs import Command
 from poezio.decorators import refresh_wrapper, command_args_parser
@@ -40,7 +40,14 @@ from poezio.roster import roster
 from poezio.theming import get_theme, dump_tuple
 from poezio.user import User
 from poezio.core.structs import Completion, Status
-from poezio.ui.types import BaseMessage, Message, InfoMessage, StatusMessage
+from poezio.ui.types import (
+    BaseMessage,
+    InfoMessage,
+    Message,
+    MucOwnJoinMessage,
+    MucOwnLeaveMessage,
+    StatusMessage,
+)
 
 log = logging.getLogger(__name__)
 
@@ -84,8 +91,6 @@ class MucTab(ChatTab):
         self.self_ping_event = None
         # UI stuff
         self.topic_win = windows.Topic()
-        self.text_win = windows.TextWin()
-        self._text_buffer.add_window(self.text_win)
         self.v_separator = windows.VerticalSeparator()
         self.user_win = windows.UserList()
         self.info_header = windows.MucInfoWin()
@@ -151,10 +156,10 @@ class MucTab(ChatTab):
         """
         status = self.core.get_status()
         if self.last_connection:
-            delta = datetime.now() - self.last_connection
+            delta = to_utc(datetime.now()) - to_utc(self.last_connection)
             seconds = delta.seconds + delta.days * 24 * 3600
         else:
-            seconds = None
+            seconds = self._text_buffer.find_last_message()
         muc.join_groupchat(
             self.core,
             self.jid.bare,
@@ -163,7 +168,6 @@ class MucTab(ChatTab):
             status=status.message,
             show=status.show,
             seconds=seconds)
-        asyncio.ensure_future(mam.on_tab_open(self))
 
     def leave_room(self, message: str):
         if self.joined:
@@ -200,7 +204,7 @@ class MucTab(ChatTab):
                            'color_spec': spec_col,
                            'nick': self.own_nick,
                        }
-            self.add_message(InfoMessage(msg), typ=2)
+            self.add_message(MucOwnLeaveMessage(msg), typ=2)
             self.disconnect()
             muc.leave_groupchat(self.core.xmpp, self.jid.bare, self.own_nick,
                                 message)
@@ -567,7 +571,7 @@ class MucTab(ChatTab):
                               'nick_col': color,
                               'info_col': info_col,
                           }
-        self.add_message(InfoMessage(enable_message), typ=2)
+        self.add_message(MucOwnJoinMessage(enable_message), typ=2)
         self.core.enable_private_tabs(self.jid.bare, enable_message)
         if '201' in status_codes:
             self.add_message(
@@ -594,6 +598,7 @@ class MucTab(ChatTab):
                     },
                 ),
                 typ=0)
+        mam.schedule_tab_open(self)
 
     def handle_presence_joined(self, presence: Presence, status_codes) -> None:
         """
@@ -651,7 +656,7 @@ class MucTab(ChatTab):
     def on_non_member_kicked(self):
         """We have been kicked because the MUC is members-only"""
         self.add_message(
-            InfoMessage(
+            MucOwnLeaveMessage(
                 'You have been kicked because you '
                 'are not a member and the room is now members-only.'
             ),
@@ -661,7 +666,7 @@ class MucTab(ChatTab):
     def on_muc_shutdown(self):
         """We have been kicked because the MUC service is shutting down"""
         self.add_message(
-            InfoMessage(
+            MucOwnLeaveMessage(
                 'You have been kicked because the'
                 ' MUC service is shutting down.'
             ),
@@ -759,6 +764,7 @@ class MucTab(ChatTab):
         """
         When someone is banned from a muc
         """
+        cls = InfoMessage
         self.users.remove(user)
         by = presence.xml.find('{%s}x/{%s}item/{%s}actor' %
                                (NS_MUC_USER, NS_MUC_USER, NS_MUC_USER))
@@ -774,6 +780,7 @@ class MucTab(ChatTab):
         char_kick = theme.CHAR_KICK
 
         if from_nick == self.own_nick:  # we are banned
+            cls = MucOwnLeaveMessage
             if by:
                 kick_msg = ('\x191}%(spec)s \x193}You\x19%(info_col)s}'
                             ' have been banned by \x194}%(by)s') % {
@@ -834,12 +841,13 @@ class MucTab(ChatTab):
                              'reason': reason.text,
                              'info_col': info_col
                          }
-        self.add_message(InfoMessage(kick_msg), typ=2)
+        self.add_message(cls(kick_msg), typ=2)
 
     def on_user_kicked(self, presence, user, from_nick):
         """
         When someone is kicked from a muc
         """
+        cls = InfoMessage
         self.users.remove(user)
         actor_elem = presence.xml.find('{%s}x/{%s}item/{%s}actor' %
                                        (NS_MUC_USER, NS_MUC_USER, NS_MUC_USER))
@@ -852,6 +860,7 @@ class MucTab(ChatTab):
         if actor_elem is not None:
             by = actor_elem.get('nick') or actor_elem.get('jid')
         if from_nick == self.own_nick:  # we are kicked
+            cls = MucOwnLeaveMessage
             if by:
                 kick_msg = ('\x191}%(spec)s \x193}You\x19'
                             '%(info_col)s} have been kicked'
@@ -912,7 +921,7 @@ class MucTab(ChatTab):
                              'reason': reason.text,
                              'info_col': info_col
                          }
-        self.add_message(InfoMessage(kick_msg), typ=2)
+        self.add_message(cls(kick_msg), typ=2)
 
     def on_user_leave_groupchat(self,
                                 user: User,
diff --git a/poezio/tabs/privatetab.py b/poezio/tabs/privatetab.py
index b43294a1..cd2123f3 100644
--- a/poezio/tabs/privatetab.py
+++ b/poezio/tabs/privatetab.py
@@ -46,8 +46,6 @@ class PrivateTab(OneToOneTab):
     def __init__(self, core, jid, nick):
         OneToOneTab.__init__(self, core, jid)
         self.own_nick = nick
-        self.text_win = windows.TextWin()
-        self._text_buffer.add_window(self.text_win)
         self.info_header = windows.PrivateInfoWin()
         self.input = windows.MessageInput()
         # keys
diff --git a/poezio/text_buffer.py b/poezio/text_buffer.py
index 3b3ac051..ff853a67 100644
--- a/poezio/text_buffer.py
+++ b/poezio/text_buffer.py
@@ -12,6 +12,7 @@ import logging
 log = logging.getLogger(__name__)
 
 from typing import (
+    cast,
     Dict,
     List,
     Optional,
@@ -19,9 +20,15 @@ from typing import (
     Tuple,
     Union,
 )
+from dataclasses import dataclass
 from datetime import datetime
 from poezio.config import config
-from poezio.ui.types import Message, BaseMessage
+from poezio.ui.types import (
+    BaseMessage,
+    Message,
+    MucOwnJoinMessage,
+    MucOwnLeaveMessage,
+)
 
 if TYPE_CHECKING:
     from poezio.windows.text_win import TextWin
@@ -35,6 +42,15 @@ class AckError(Exception):
     pass
 
 
+@dataclass
+class HistoryGap:
+    """Class representing a period of non-presence inside a MUC"""
+    leave_message: Optional[BaseMessage]
+    join_message: Optional[BaseMessage]
+    last_timestamp_before_leave: Optional[datetime]
+    first_timestamp_after_join: Optional[datetime]
+
+
 class TextBuffer:
     """
     This class just keep trace of messages, in a list with various
@@ -44,7 +60,7 @@ class TextBuffer:
     def __init__(self, messages_nb_limit: Optional[int] = None) -> None:
 
         if messages_nb_limit is None:
-            messages_nb_limit = config.get('max_messages_in_memory')
+            messages_nb_limit = cast(int, config.get('max_messages_in_memory'))
         self._messages_nb_limit = messages_nb_limit  # type: int
         # Message objects
         self.messages = []  # type: List[BaseMessage]
@@ -58,6 +74,99 @@ class TextBuffer:
     def add_window(self, win) -> None:
         self._windows.append(win)
 
+    def find_last_gap_muc(self) -> Optional[HistoryGap]:
+        """Find the last known history gap contained in buffer"""
+        leave = None  # type:Optional[Tuple[int, BaseMessage]]
+        join = None  # type:Optional[Tuple[int, BaseMessage]]
+        for i, item in enumerate(reversed(self.messages)):
+            if isinstance(item, MucOwnLeaveMessage):
+                leave = (len(self.messages) - i - 1, item)
+                break
+            elif join and isinstance(item, MucOwnJoinMessage):
+                leave = (len(self.messages) - i - 1, item)
+                break
+            if isinstance(item, MucOwnJoinMessage):
+                join = (len(self.messages) - i - 1, item)
+
+        last_timestamp = None
+        first_timestamp = datetime.now()
+
+        # Identify the special case when we got disconnected from a chatroom
+        # without receiving or sending the relevant presence, therefore only
+        # having two joins with no leave, and messages in the middle.
+        if leave and join and isinstance(leave[1], MucOwnJoinMessage):
+            for i in range(join[0] - 1, leave[0], - 1):
+                if isinstance(self.messages[i], Message):
+                    leave = (
+                        i,
+                        self.messages[i]
+                    )
+                    last_timestamp = self.messages[i].time
+                    break
+        # If we have a normal gap but messages inbetween, it probably
+        # already has history, so abort there without returning it.
+        if join and leave:
+            for i in range(leave[0] + 1, join[0], 1):
+                if isinstance(self.messages[i], Message):
+                    return None
+        elif not (join or leave):
+            return None
+
+        # If a leave message is found, get the last Message timestamp
+        # before it.
+        if leave is None:
+            leave_msg = None
+        elif last_timestamp is None:
+            leave_msg = leave[1]
+            for i in range(leave[0], 0, -1):
+                if isinstance(self.messages[i], Message):
+                    last_timestamp = self.messages[i].time
+                    break
+        else:
+            leave_msg = leave[1]
+        # If a join message is found, get the first Message timestamp
+        # after it, or the current time.
+        if join is None:
+            join_msg = None
+        else:
+            join_msg = join[1]
+            for i in range(join[0], len(self.messages)):
+                msg = self.messages[i]
+                if isinstance(msg, Message) and msg.time < first_timestamp:
+                    first_timestamp = msg.time
+                    break
+        return HistoryGap(
+            leave_message=leave_msg,
+            join_message=join_msg,
+            last_timestamp_before_leave=last_timestamp,
+            first_timestamp_after_join=first_timestamp,
+        )
+
+    def get_gap_index(self, gap: HistoryGap) -> Optional[int]:
+        """Find the first index to insert into inside a gap"""
+        if gap.leave_message is None:
+            return 0
+        for i, msg in enumerate(self.messages):
+            if msg is gap.leave_message:
+                return i + 1
+        return None
+
+    def add_history_messages(self, messages: List[BaseMessage], gap: Optional[HistoryGap] = None) -> None:
+        """Insert history messages at their correct place """
+        index = 0
+        new_index = None
+        if gap is not None:
+            new_index = self.get_gap_index(gap)
+            if new_index is None:  # Not sure what happened, abort
+                return
+            index = new_index
+        for message in messages:
+            self.messages.insert(index, message)
+            index += 1
+            log.debug('inserted message: %s', message)
+        for window in self._windows:  # make the associated windows
+            window.rebuild_everything(self)
+
     @property
     def last_message(self) -> Optional[BaseMessage]:
         return self.messages[-1] if self.messages else None
@@ -72,8 +181,8 @@ class TextBuffer:
             self.messages.pop(0)
 
         ret_val = 0
-        show_timestamps = config.get('show_timestamps')
-        nick_size = config.get('max_nick_length')
+        show_timestamps = cast(bool, config.get('show_timestamps'))
+        nick_size = cast(int, config.get('max_nick_length'))
         for window in self._windows:  # make the associated windows
             # build the lines from the new message
             nb = window.build_new_message(
@@ -82,8 +191,7 @@ class TextBuffer:
                 nick_size=nick_size)
             if ret_val == 0:
                 ret_val = nb
-            top = isinstance(msg, Message) and msg.top
-            if window.pos != 0 and top is False:
+            if window.pos != 0:
                 window.scroll_up(nb)
 
         return min(ret_val, 1)
@@ -197,6 +305,13 @@ class TextBuffer:
     def del_window(self, win) -> None:
         self._windows.remove(win)
 
+    def find_last_message(self) -> Optional[Message]:
+        """Find the last real message received in this buffer"""
+        for message in reversed(self.messages):
+            if isinstance(message, Message):
+                return message
+        return None
+
     def __del__(self):
         size = len(self.messages)
         log.debug('** Deleting %s messages from textbuffer', size)
diff --git a/poezio/ui/render.py b/poezio/ui/render.py
index a431b4e7..c85d3cc5 100644
--- a/poezio/ui/render.py
+++ b/poezio/ui/render.py
@@ -94,8 +94,6 @@ def build_message(msg: Message, width: int, timestamp: bool, nick_size: int = 10
     offset = msg.compute_offset(timestamp, nick_size)
     lines = poopt.cut_text(txt, width - offset - 1)
     generated_lines = generate_lines(lines, msg, default_color='')
-    if msg.top:
-        generated_lines.reverse()
     return generated_lines
 
 
diff --git a/poezio/ui/types.py b/poezio/ui/types.py
index ae72b6b9..15117275 100644
--- a/poezio/ui/types.py
+++ b/poezio/ui/types.py
@@ -12,6 +12,7 @@ from poezio.ui.consts import (
 )
 
 
+
 class BaseMessage:
     __slots__ = ('txt', 'time', 'identifier')
 
@@ -27,12 +28,24 @@ class BaseMessage:
         return SHORT_FORMAT_LENGTH + 1
 
 
+class EndOfArchive(BaseMessage):
+    """Marker added to a buffer when we reach the end of a MAM archive"""
+
+
 class InfoMessage(BaseMessage):
     def __init__(self, txt: str, identifier: str = '', time: Optional[datetime] = None):
         txt = ('\x19%s}' % dump_tuple(get_theme().COLOR_INFORMATION_TEXT)) + txt
         super().__init__(txt=txt, identifier=identifier, time=time)
 
 
+class MucOwnLeaveMessage(InfoMessage):
+    """Status message displayed on our room leave/kick/ban"""
+
+
+class MucOwnJoinMessage(InfoMessage):
+    """Status message displayed on our room join"""
+
+
 class XMLLog(BaseMessage):
     """XML Log message"""
     __slots__ = ('txt', 'time', 'identifier', 'incoming')
diff --git a/poezio/windows/text_win.py b/poezio/windows/text_win.py
index 9e6641f7..2cb75271 100644
--- a/poezio/windows/text_win.py
+++ b/poezio/windows/text_win.py
@@ -95,14 +95,10 @@ class TextWin(Win):
         lines = build_lines(
             message, self.width, timestamp=timestamp, nick_size=nick_size
         )
-        if isinstance(message, Message) and message.top:
-            for line in lines:
-                self.built_lines.insert(0, line)
+        if self.lock:
+            self.lock_buffer.extend(lines)
         else:
-            if self.lock:
-                self.lock_buffer.extend(lines)
-            else:
-                self.built_lines.extend(lines)
+            self.built_lines.extend(lines)
         if not lines or not lines[0]:
             return 0
         if isinstance(message, Message) and message.highlight:
diff --git a/test/test_text_buffer.py b/test/test_text_buffer.py
new file mode 100644
index 00000000..65c6d9bf
--- /dev/null
+++ b/test/test_text_buffer.py
@@ -0,0 +1,198 @@
+"""
+Tests for the TextBuffer class
+"""
+from pytest import fixture
+
+from poezio.text_buffer import (
+    TextBuffer,
+    HistoryGap,
+)
+
+from poezio.ui.types import (
+    Message,
+    BaseMessage,
+    MucOwnJoinMessage,
+    MucOwnLeaveMessage,
+)
+
+
+@fixture(scope='function')
+def buf2048():
+    return TextBuffer(2048)
+
+@fixture(scope='function')
+def msgs_nojoin():
+    msg1 = Message('1', 'q')
+    msg2 = Message('2', 's')
+    leave = MucOwnLeaveMessage('leave')
+    return [msg1, msg2, leave]
+
+
+@fixture(scope='function')
+def msgs_noleave():
+    join = MucOwnJoinMessage('join')
+    msg3 = Message('3', 'd')
+    msg4 = Message('4', 'f')
+    return [join, msg3, msg4]
+
+@fixture(scope='function')
+def msgs_doublejoin():
+    join = MucOwnJoinMessage('join')
+    msg1 = Message('1', 'd')
+    msg2 = Message('2', 'f')
+    join2 = MucOwnJoinMessage('join')
+    return [join, msg1, msg2, join2]
+
+def test_last_message(buf2048):
+    msg = BaseMessage('toto')
+    buf2048.add_message(BaseMessage('titi'))
+    buf2048.add_message(msg)
+    assert buf2048.last_message is msg
+
+
+def test_message_nb_limit():
+    buf = TextBuffer(5)
+    for i in range(10):
+        buf.add_message(BaseMessage("%s" % i))
+    assert len(buf.messages) == 5
+
+
+def test_find_gap(buf2048, msgs_noleave):
+    msg1 = Message('1', 'q')
+    msg2 = Message('2', 's')
+    leave = MucOwnLeaveMessage('leave')
+    join = MucOwnJoinMessage('join')
+    msg3 = Message('3', 'd')
+    msg4 = Message('4', 'f')
+    msgs = [msg1, msg2, leave, join, msg3, msg4]
+    for msg in msgs:
+        buf2048.add_message(msg)
+    gap = buf2048.find_last_gap_muc()
+    assert gap.leave_message == leave
+    assert gap.join_message == join
+    assert gap.last_timestamp_before_leave == msg2.time
+    assert gap.first_timestamp_after_join == msg3.time
+
+
+def test_find_gap_doublejoin(buf2048, msgs_doublejoin):
+    for msg in msgs_doublejoin:
+        buf2048.add_message(msg)
+    gap = buf2048.find_last_gap_muc()
+    assert gap.leave_message == msgs_doublejoin[2]
+    assert gap.join_message == msgs_doublejoin[3]
+
+
+def test_find_gap_doublejoin_no_msg(buf2048):
+    join1 = MucOwnJoinMessage('join')
+    join2 = MucOwnJoinMessage('join')
+    for msg in [join1, join2]:
+        buf2048.add_message(msg)
+    gap = buf2048.find_last_gap_muc()
+    assert gap.leave_message is join1
+    assert gap.join_message is join2
+
+
+def test_find_gap_already_filled(buf2048):
+    msg1 = Message('1', 'q')
+    msg2 = Message('2', 's')
+    leave = MucOwnLeaveMessage('leave')
+    msg5 = Message('5', 'g')
+    msg6 = Message('6', 'h')
+    join = MucOwnJoinMessage('join')
+    msg3 = Message('3', 'd')
+    msg4 = Message('4', 'f')
+    msgs = [msg1, msg2, leave, msg5, msg6, join, msg3, msg4]
+    for msg in msgs:
+        buf2048.add_message(msg)
+    assert buf2048.find_last_gap_muc() is None
+
+
+def test_find_gap_noleave(buf2048, msgs_noleave):
+    for msg in msgs_noleave:
+        buf2048.add_message(msg)
+    gap = buf2048.find_last_gap_muc()
+    assert gap.leave_message is None
+    assert gap.last_timestamp_before_leave is None
+    assert gap.join_message == msgs_noleave[0]
+    assert gap.first_timestamp_after_join == msgs_noleave[1].time
+
+
+def test_find_gap_nojoin(buf2048, msgs_nojoin):
+    for msg in msgs_nojoin:
+        buf2048.add_message(msg)
+    gap = buf2048.find_last_gap_muc()
+    assert gap.leave_message == msgs_nojoin[-1]
+    assert gap.join_message is None
+    assert gap.last_timestamp_before_leave == msgs_nojoin[1].time
+
+
+def test_get_gap_index(buf2048):
+    msg1 = Message('1', 'q')
+    msg2 = Message('2', 's')
+    leave = MucOwnLeaveMessage('leave')
+    join = MucOwnJoinMessage('join')
+    msg3 = Message('3', 'd')
+    msg4 = Message('4', 'f')
+    msgs = [msg1, msg2, leave, join, msg3, msg4]
+    for msg in msgs:
+        buf2048.add_message(msg)
+    gap = buf2048.find_last_gap_muc()
+    assert buf2048.get_gap_index(gap) == 3
+
+
+def test_get_gap_index_doublejoin(buf2048, msgs_doublejoin):
+    for msg in msgs_doublejoin:
+        buf2048.add_message(msg)
+    gap = buf2048.find_last_gap_muc()
+    assert buf2048.get_gap_index(gap) == 3
+
+
+def test_get_gap_index_doublejoin_no_msg(buf2048):
+    join1 = MucOwnJoinMessage('join')
+    join2 = MucOwnJoinMessage('join')
+    for msg in [join1, join2]:
+        buf2048.add_message(msg)
+    gap = buf2048.find_last_gap_muc()
+    assert buf2048.get_gap_index(gap) == 1
+
+
+def test_get_gap_index_nojoin(buf2048, msgs_nojoin):
+    for msg in msgs_nojoin:
+        buf2048.add_message(msg)
+    gap = buf2048.find_last_gap_muc()
+    assert buf2048.get_gap_index(gap) == 3
+
+
+def test_get_gap_index_noleave(buf2048, msgs_noleave):
+    for msg in msgs_noleave:
+        buf2048.add_message(msg)
+    gap = buf2048.find_last_gap_muc()
+    assert buf2048.get_gap_index(gap) == 0
+
+
+def test_add_history_messages(buf2048):
+    msg1 = Message('1', 'q')
+    msg2 = Message('2', 's')
+    leave = MucOwnLeaveMessage('leave')
+    join = MucOwnJoinMessage('join')
+    msg3 = Message('3', 'd')
+    msg4 = Message('4', 'f')
+    msgs = [msg1, msg2, leave, join, msg3, msg4]
+    for msg in msgs:
+        buf2048.add_message(msg)
+    msg5 = Message('5', 'g')
+    msg6 = Message('6', 'h')
+    gap = buf2048.find_last_gap_muc()
+    buf2048.add_history_messages([msg5, msg6], gap=gap)
+    assert buf2048.messages == [msg1, msg2, leave, msg5, msg6, join, msg3, msg4]
+
+
+def test_add_history_empty(buf2048):
+    msg1 = Message('1', 'q')
+    msg2 = Message('2', 's')
+    msg3 = Message('3', 'd')
+    msg4 = Message('4', 'f')
+    buf2048.add_message(msg1)
+    buf2048.add_history_messages([msg2, msg3, msg4])
+    assert buf2048.messages == [msg2, msg3, msg4, msg1]
+