Resolve xml:lang issue with duplicated elements depending on ordering.

2 files changed, 365 insertions, 43 deletions

diff --git a/sleekxmpp/xmlstream/stanzabase.py b/sleekxmpp/xmlstream/stanzabase.py
index 88276ddf..4af441cc 100644
--- a/sleekxmpp/xmlstream/stanzabase.py
+++ b/sleekxmpp/xmlstream/stanzabase.py
@@ -421,12 +421,6 @@ class ElementBase(object):
         #: ``'{namespace}elementname'``.
         self.tag = self.tag_name()
 
-        if 'lang' not in self.interfaces:
-            if isinstance(self.interfaces, tuple):
-                self.interfaces += ('lang',)
-            else:
-                self.interfaces.add('lang')
-
         #: A :class:`weakref.weakref` to the parent stanza, if there is one.
         #: If not, then :attr:`parent` is ``None``.
         self.parent = None
@@ -574,6 +568,7 @@ class ElementBase(object):
         .. versionadded:: 1.0-Beta1
         """
         values = {}
+        values['lang'] = self['lang']
         for interface in self.interfaces:
             values[interface] = self[interface]
             if interface in self.lang_interfaces:
@@ -629,6 +624,8 @@ class ElementBase(object):
                                 sub.values = subdict
                                 self.iterables.append(sub)
                                 break
+            elif interface == 'lang':
+                self[interface] = value
             elif interface in self.interfaces:
                 self[full_interface] = value
             elif interface in self.plugin_attrib_map:
@@ -678,7 +675,7 @@ class ElementBase(object):
 
         if attrib == 'substanzas':
             return self.iterables
-        elif attrib in self.interfaces:
+        elif attrib in self.interfaces or attrib == 'lang':
             get_method = "get_%s" % attrib.lower()
             get_method2 = "get%s" % attrib.title()
 
@@ -752,7 +749,7 @@ class ElementBase(object):
         if lang and attrib in self.lang_interfaces:
             kwargs['lang'] = lang
 
-        if attrib in self.interfaces:
+        if attrib in self.interfaces or attrib == 'lang':
             if value is not None:
                 set_method = "set_%s" % attrib.lower()
                 set_method2 = "set%s" % attrib.title()
@@ -838,7 +835,7 @@ class ElementBase(object):
         if lang and attrib in self.lang_interfaces:
             kwargs['lang'] = lang
 
-        if attrib in self.interfaces:
+        if attrib in self.interfaces or attrib == 'lang':
             del_method = "del_%s" % attrib.lower()
             del_method2 = "del%s" % attrib.title()
 
@@ -973,10 +970,6 @@ class ElementBase(object):
         :param keep: Indicates if the element should be kept if its text is
                      removed. Defaults to False.
         """
-        path = self._fix_ns(name, split=True)
-        element = self.xml.find(name)
-        parent = self.xml
-
         default_lang = self.get_lang()
         if lang is None:
             lang = default_lang
@@ -984,32 +977,51 @@ class ElementBase(object):
         if not text and not keep:
             return self._del_sub(name, lang=lang)
 
-        if element is None:
-            # We need to add the element. If the provided name was
-            # an XPath expression, some of the intermediate elements
-            # may already exist. If so, we want to use those instead
-            # of generating new elements.
-            last_xml = self.xml
-            walked = []
-            for ename in path:
-                walked.append(ename)
-                element = self.xml.find("/".join(walked))
-                if element is None:
-                    element = ET.Element(ename)
-                    if lang:
-                        element.attrib['{%s}lang' % XML_NS] = lang
-                    last_xml.append(element)
-                parent = last_xml
-                last_xml = element
-            element = last_xml
+        path = self._fix_ns(name, split=True)
+        name = path[-1]
+        parent = self.xml
 
-        if lang:
-            if element.attrib.get('{%s}lang' % XML_NS, default_lang) != lang:
-                element = ET.Element(ename)
-                element.attrib['{%s}lang' % XML_NS] = lang
-                parent.append(element)
+        # The first goal is to find the parent of the subelement, or, if
+        # we can't find that, the closest grandparent element.
+        missing_path = []
+        search_order = path[:-1]
+        while search_order:
+            parent = self.xml.find('/'.join(search_order))
+            ename = search_order.pop()
+            if parent is not None:
+                break
+            else:
+                missing_path.append(ename)
+        missing_path.reverse()
+
+        # Find all existing elements that match the desired
+        # element path (there may be multiples due to different
+        # languages values).
+        if parent is not None:
+            elements = self.xml.findall('/'.join(path))
+        else:
+            parent = self.xml
+            elements = []
+
+        # Insert the remaining grandparent elements that don't exist yet.
+        for ename in missing_path:
+            element = ET.Element(ename)
+            parent.append(element)
+            parent = element
+
+        # Re-use an existing element with the proper language, if one exists.
+        for element in elements:
+            elang = element.attrib.get('{%s}lang' % XML_NS, default_lang)
+            if not lang and elang == default_lang or lang and lang == elang:
+                element.text = text
+                return element
 
+        # No useable element exists, so create a new one.
+        element = ET.Element(name)
         element.text = text
+        if lang and lang != default_lang:
+            element.attrib['{%s}lang' % XML_NS] = lang
+        parent.append(element)
         return element
 
     def _set_all_sub_text(self, name, values, keep=False, lang=None):
@@ -1184,6 +1196,7 @@ class ElementBase(object):
         out = []
         out += [x for x in self.interfaces]
         out += [x for x in self.loaded_plugins]
+        out.append('lang')
         if self.iterables:
             out.append('substanzas')
         return out
@@ -1263,7 +1276,7 @@ class ElementBase(object):
         """
         return "{%s}%s" % (cls.namespace, cls.name)
 
-    def get_lang(self):
+    def get_lang(self, lang=None):
         result = self.xml.attrib.get('{%s}lang' % XML_NS, '')
         if not result and self.parent and self.parent():
             return self.parent()['lang']
diff --git a/tests/test_stanza_element.py b/tests/test_stanza_element.py
index 1b47e733..b7ccdb87 100644
--- a/tests/test_stanza_element.py
+++ b/tests/test_stanza_element.py
@@ -1,5 +1,6 @@
 from sleekxmpp.test import *
 from sleekxmpp.xmlstream.stanzabase import ElementBase
+from sleekxmpp.thirdparty import OrderedDict
 
 
 class TestElementBase(SleekTest):
@@ -760,7 +761,7 @@ class TestElementBase(SleekTest):
           <foo xmlns="foo" />
         """)
 
-        self.assertFalse(stanza['bar'], 
+        self.assertFalse(stanza['bar'],
                 "Returned True for missing bool interface element.")
 
         stanza['bar'] = True
@@ -797,7 +798,7 @@ class TestElementBase(SleekTest):
             namespace = 'baz'
             plugin_attrib = name
             plugin_multi_attrib = 'bazs'
-           
+
         register_stanza_plugin(TestStanza, TestMultiStanza1, iterable=True)
         register_stanza_plugin(TestStanza, TestMultiStanza2, iterable=True)
 
@@ -829,9 +830,9 @@ class TestElementBase(SleekTest):
               <baz xmlns="baz" />
             """)
 
-        self.assertEqual(len(bars), 2, 
+        self.assertEqual(len(bars), 2,
                 "Wrong number of <bar /> stanzas: %s" % len(bars))
-        self.assertEqual(len(bazs), 2, 
+        self.assertEqual(len(bazs), 2,
                 "Wrong number of <baz /> stanzas: %s" % len(bazs))
 
     def testSetMultiAttrib(self):
@@ -853,7 +854,7 @@ class TestElementBase(SleekTest):
             namespace = 'baz'
             plugin_attrib = name
             plugin_multi_attrib = 'bazs'
-           
+
         register_stanza_plugin(TestStanza, TestMultiStanza1, iterable=True)
         register_stanza_plugin(TestStanza, TestMultiStanza2, iterable=True)
 
@@ -906,7 +907,7 @@ class TestElementBase(SleekTest):
             namespace = 'baz'
             plugin_attrib = name
             plugin_multi_attrib = 'bazs'
-           
+
         register_stanza_plugin(TestStanza, TestMultiStanza1, iterable=True)
         register_stanza_plugin(TestStanza, TestMultiStanza2, iterable=True)
 
@@ -938,5 +939,313 @@ class TestElementBase(SleekTest):
         self.assertEqual(len(stanza['substanzas']), 2,
             "Wrong number of substanzas: %s" % len(stanza['substanzas']))
 
+    def testDefaultLang(self):
+        """Test setting a normal subinterface when a default language is set"""
+
+        class TestStanza(ElementBase):
+            name = 'foo'
+            namespace = 'test'
+            interfaces = set(['test'])
+            sub_interfaces = interfaces
+            lang_interfaces = interfaces
+
+        stanza = TestStanza()
+        stanza['lang'] = 'sv'
+        stanza['test'] = 'hej'
+
+        self.check(stanza, """
+          <foo xmlns="test" xml:lang="sv">
+            <test>hej</test>
+          </foo>
+        """)
+
+        self.assertEqual(stanza['test'], 'hej',
+                "Incorrect subinterface value: %s" % stanza['test'])
+
+        self.assertEqual(stanza['test|sv'], 'hej',
+                "Incorrect subinterface value: %s" % stanza['test|sv'])
+
+    def testSpecifyLangWithDefault(self):
+        """Test specifying various languages."""
+
+        class TestStanza(ElementBase):
+            name = 'foo'
+            namespace = 'test'
+            interfaces = set(['test'])
+            sub_interfaces = interfaces
+            lang_interfaces = interfaces
+
+        stanza = TestStanza()
+        stanza['lang'] = 'sv'
+        stanza['test'] = 'hej'
+        stanza['test|en'] = 'hi'
+        stanza['test|es'] = 'hola'
+
+        self.check(stanza, """
+          <foo xmlns="test" xml:lang="sv">
+            <test>hej</test>
+            <test xml:lang="en">hi</test>
+            <test xml:lang="es">hola</test>
+          </foo>
+        """)
+
+        self.assertEqual(stanza['test'], 'hej',
+                "Incorrect subinterface value: %s" % stanza['test'])
+
+        self.assertEqual(stanza['test|sv'], 'hej',
+                "Incorrect subinterface value: %s" % stanza['test|sv'])
+
+        self.assertEqual(stanza['test|en'], 'hi',
+                "Incorrect subinterface value: %s" % stanza['test|en'])
+
+        self.assertEqual(stanza['test|es'], 'hola',
+                "Incorrect subinterface value: %s" % stanza['test|es'])
+
+    def testSpecifyLangWithNoDefault(self):
+        """Test specifying various languages."""
+
+        class TestStanza(ElementBase):
+            name = 'foo'
+            namespace = 'test'
+            interfaces = set(['test'])
+            sub_interfaces = interfaces
+            lang_interfaces = interfaces
+
+        stanza = TestStanza()
+        stanza['test'] = 'hej'
+        stanza['test|en'] = 'hi'
+        stanza['test|es'] = 'hola'
+
+        self.check(stanza, """
+          <foo xmlns="test">
+            <test>hej</test>
+            <test xml:lang="en">hi</test>
+            <test xml:lang="es">hola</test>
+          </foo>
+        """)
+
+        self.assertEqual(stanza['test'], 'hej',
+                "Incorrect subinterface value: %s" % stanza['test'])
+
+        self.assertEqual(stanza['test|en'], 'hi',
+                "Incorrect subinterface value: %s" % stanza['test|en'])
+
+        self.assertEqual(stanza['test|es'], 'hola',
+                "Incorrect subinterface value: %s" % stanza['test|es'])
+
+    def testModifyLangInterfaceWithDefault(self):
+        """Test resetting an interface when a default lang is used."""
+
+        class TestStanza(ElementBase):
+            name = 'foo'
+            namespace = 'test'
+            interfaces = set(['test'])
+            sub_interfaces = interfaces
+            lang_interfaces = interfaces
+
+        stanza = TestStanza()
+        stanza['lang'] = 'es'
+        stanza['test'] = 'hola'
+        stanza['test|en'] = 'hi'
+
+        self.check(stanza, """
+          <foo xmlns="test" xml:lang="es">
+            <test>hola</test>
+            <test xml:lang="en">hi</test>
+          </foo>
+        """)
+
+        stanza['test'] = 'adios'
+        stanza['test|en'] = 'bye'
+
+        self.check(stanza, """
+          <foo xmlns="test" xml:lang="es">
+            <test>adios</test>
+            <test xml:lang="en">bye</test>
+          </foo>
+        """)
+
+        self.assertEqual(stanza['test'], 'adios',
+                "Incorrect subinterface value: %s" % stanza['test'])
+
+        self.assertEqual(stanza['test|es'], 'adios',
+                "Incorrect subinterface value: %s" % stanza['test|es'])
+
+        self.assertEqual(stanza['test|en'], 'bye',
+                "Incorrect subinterface value: %s" % stanza['test|en'])
+
+        stanza['test|es'] = 'hola'
+
+        self.check(stanza, """
+          <foo xmlns="test" xml:lang="es">
+            <test>hola</test>
+            <test xml:lang="en">bye</test>
+          </foo>
+        """)
+
+        self.assertEqual(stanza['test'], 'hola',
+                "Incorrect subinterface value: %s" % stanza['test'])
+
+        self.assertEqual(stanza['test|es'], 'hola',
+                "Incorrect subinterface value: %s" % stanza['test|es'])
+
+    def testModifyLangInterfaceWithNoDefault(self):
+        """Test resetting an interface when no default lang is used."""
+
+        class TestStanza(ElementBase):
+            name = 'foo'
+            namespace = 'test'
+            interfaces = set(['test'])
+            sub_interfaces = interfaces
+            lang_interfaces = interfaces
+
+        stanza = TestStanza()
+        stanza['test'] = 'hola'
+        stanza['test|en'] = 'hi'
+
+        self.check(stanza, """
+          <foo xmlns="test">
+            <test>hola</test>
+            <test xml:lang="en">hi</test>
+          </foo>
+        """)
+
+        stanza['test'] = 'adios'
+        stanza['test|en'] = 'bye'
+
+        self.check(stanza, """
+          <foo xmlns="test">
+            <test>adios</test>
+            <test xml:lang="en">bye</test>
+          </foo>
+        """)
+
+        self.assertEqual(stanza['test'], 'adios',
+                "Incorrect subinterface value: %s" % stanza['test'])
+
+        self.assertEqual(stanza['test'], 'adios',
+                "Incorrect subinterface value: %s" % stanza['test|es'])
+
+        self.assertEqual(stanza['test|en'], 'bye',
+                "Incorrect subinterface value: %s" % stanza['test|en'])
+
+    def testDelInterfacesWithDefaultLang(self):
+        """Test deleting interfaces with a default lang set."""
+
+        class TestStanza(ElementBase):
+            name = 'foo'
+            namespace = 'test'
+            interfaces = set(['test'])
+            sub_interfaces = interfaces
+            lang_interfaces = interfaces
+
+        stanza = TestStanza()
+        stanza['lang'] = 'en'
+        stanza['test'] = 'hi'
+        stanza['test|no'] = 'hej'
+        stanza['test|fr'] = 'bonjour'
+
+        self.check(stanza, """
+          <foo xmlns="test" xml:lang="en">
+            <test>hi</test>
+            <test xml:lang="no">hej</test>
+            <test xml:lang="fr">bonjour</test>
+          </foo>
+        """)
+
+        del stanza['test']
+
+        self.check(stanza, """
+          <foo xmlns="test" xml:lang="en">
+            <test xml:lang="no">hej</test>
+            <test xml:lang="fr">bonjour</test>
+          </foo>
+        """)
+
+        del stanza['test|no']
+
+        self.check(stanza, """
+          <foo xmlns="test" xml:lang="en">
+            <test xml:lang="fr">bonjour</test>
+          </foo>
+        """)
+
+    def testDelInterfacesWithNoDefaultLang(self):
+        """Test deleting interfaces with no default lang set."""
+
+        class TestStanza(ElementBase):
+            name = 'foo'
+            namespace = 'test'
+            interfaces = set(['test'])
+            sub_interfaces = interfaces
+            lang_interfaces = interfaces
+
+        stanza = TestStanza()
+        stanza['test'] = 'hi'
+        stanza['test|no'] = 'hej'
+        stanza['test|fr'] = 'bonjour'
+
+        self.check(stanza, """
+          <foo xmlns="test">
+            <test>hi</test>
+            <test xml:lang="no">hej</test>
+            <test xml:lang="fr">bonjour</test>
+          </foo>
+        """)
+
+        del stanza['test']
+
+        self.check(stanza, """
+          <foo xmlns="test">
+            <test xml:lang="no">hej</test>
+            <test xml:lang="fr">bonjour</test>
+          </foo>
+        """)
+
+        del stanza['test|no']
+
+        self.check(stanza, """
+          <foo xmlns="test">
+            <test xml:lang="fr">bonjour</test>
+          </foo>
+        """)
+
+    def testStarLang(self):
+        """Test using interface|*."""
+
+        class TestStanza(ElementBase):
+            name = 'foo'
+            namespace = 'test'
+            interfaces = set(['test'])
+            sub_interfaces = interfaces
+            lang_interfaces = interfaces
+
+        data = OrderedDict()
+        data['en'] = 'hi'
+        data['fr'] = 'bonjour'
+        data['no'] = 'hej'
+
+        stanza = TestStanza()
+        stanza['test|*'] = data
+
+        self.check(stanza, """
+          <foo xmlns="test">
+            <test xml:lang="en">hi</test>
+            <test xml:lang="fr">bonjour</test>
+            <test xml:lang="no">hej</test>
+          </foo>
+        """)
+
+        data2 = stanza['test|*']
+
+        self.assertEqual(data, data2,
+                "Did not extract expected language data: %s" % data2)
+
+        del stanza['test|*']
+
+        self.check(stanza, """
+          <foo xmlns="test" />
+        """)
+
 
 suite = unittest.TestLoader().loadTestsFromTestCase(TestElementBase)