merged branch Seldaek/trans_charset (PR #2339)

Commits
-------

5473d3b [Translation] Allow use of UTF-8 encoded catalogues into non-UTF-8 applications
deb6dea [Translation] Add failing tests to verify that UTF-8 lang files can't be used with another charset

Discussion
----------

Allow use of UTF-8 catalogues in non-UTF-8 applications

This is #2313 but targetting the master branch.

Bug fix: yes
Feature addition: ?:)
Backwards compatibility break: no
Symfony2 tests pass: yes
Fixes the following tickets: -

The problem I'm having is that, while porting an existing app, we are using UTF-8 everywhere to have a migration path ready, but the current application and DB is still in ISO-8859-1, which means translations containing accented chars are broken.

Also, we didn't hit the issue yet since we don't use forms much, but I imagine we would have similar issues with core translations for the validator which are all UTF-8 encoded.

Note that I explicitly suppressed this conversion in case your application is setup as UTF-8, to make sure most people are not affected by any slow down this introduces.

src/Symfony/Bundle/FrameworkBundle/Resources/config/translation.xml

@@ -35,6 +35,7 @@
             <argument type="collection">
                 <argument key="cache_dir">%kernel.cache_dir%/translations</argument>
                 <argument key="debug">%kernel.debug%</argument>
+                <argument key="charset">%kernel.charset%</argument>
             </argument>
             <argument type="service" id="session" on-invalid="ignore" />
         </service>

src/Symfony/Bundle/FrameworkBundle/Translation/Translator.php

@@ -46,8 +46,6 @@ class Translator extends BaseTranslator
      */
     public function __construct(ContainerInterface $container, MessageSelector $selector, $loaderIds = array(), array $options = array(), Session $session = null)
     {
-        parent::__construct(null, $selector);
-
         $this->session = $session;
         $this->container = $container;
         $this->loaderIds = $loaderIds;
@@ -55,6 +53,7 @@ class Translator extends BaseTranslator
         $this->options = array(
             'cache_dir' => null,
             'debug'     => false,
+            'charset'   => null,
         );
 
         // check option names
@@ -63,6 +62,11 @@ class Translator extends BaseTranslator
         }
 
         $this->options = array_merge($this->options, $options);
+
+        if ($this->options['charset'] === 'UTF-8') {
+            $this->options['charset'] = null;
+        }
+        parent::__construct(null, $selector, $this->options['charset']);
     }
 
     /**

src/Symfony/Component/Translation/Translator.php

@@ -28,16 +28,18 @@ class Translator implements TranslatorInterface
     private $loaders;
     private $resources;
     private $selector;
+    private $charset;
 
     /**
      * Constructor.
      *
      * @param string          $locale   The locale
      * @param MessageSelector $selector The message selector for pluralization
+     * @param string          $charset  Application charset
      *
      * @api
      */
-    public function __construct($locale, MessageSelector $selector)
+    public function __construct($locale, MessageSelector $selector, $charset = null)
     {
         $this->locale = $locale;
         $this->selector = $selector;
@@ -45,6 +47,7 @@ class Translator implements TranslatorInterface
         $this->resources = array();
         $this->catalogues = array();
         $this->fallbackLocales = array();
+        $this->charset = $charset;
     }
 
     /**
@@ -173,7 +176,18 @@ class Translator implements TranslatorInterface
                 if (!isset($this->loaders[$resource[0]])) {
                     throw new \RuntimeException(sprintf('The "%s" translation loader is not registered.', $resource[0]));
                 }
-                $this->catalogues[$locale]->addCatalogue($this->loaders[$resource[0]]->load($resource[1], $locale, $resource[2]));
+                $catalogue = $this->loaders[$resource[0]]->load($resource[1], $locale, $resource[2]);
+                if (null !== $this->charset && extension_loaded('mbstring')) {
+                    foreach ($catalogue->all() as $domain => $messages) {
+                        foreach ($messages as $key => $translation) {
+                            $srcCharset = mb_detect_encoding($translation);
+                            if ($srcCharset !== $this->charset) {
+                                $catalogue->set($key, mb_convert_encoding($translation, $this->charset, $srcCharset), $domain);
+                            }
+                        }
+                    }
+                }
+                $this->catalogues[$locale]->addCatalogue($catalogue);
             }
         }
     }

tests/Symfony/Tests/Component/Translation/TranslatorTest.php

@@ -132,6 +132,27 @@ class TranslatorTest extends \PHPUnit_Framework_TestCase
         $this->assertEquals($expected, $translator->trans($id, array(), '', 'fr'));
     }
 
+    /**
+     * @dataProvider getLoadCatalogueTests
+     */
+    public function testLoadCatalogueConvertsEncoding($translation, $charset)
+    {
+        if (!extension_loaded('mbstring')) {
+            $this->markTestSkipped('This test relies on the mbstring extension');
+        }
+        $translator = new Translator('en', new MessageSelector(), $charset);
+        $translator->addLoader('array', new ArrayLoader());
+        $translator->addResource('array', array('id' => $translation), 'en', 'messages');
+
+        if (null !== $charset && mb_detect_encoding($translation) !== $charset) {
+            $expected = mb_convert_encoding($translation, $charset, mb_detect_encoding($translation));
+        } else {
+            $expected = $translation;
+        }
+
+        $this->assertEquals($expected, $translator->trans('id', array(), 'messages', 'en'));
+    }
+
     /**
      * @dataProvider getTransChoiceTests
      */
@@ -199,6 +220,21 @@ class TranslatorTest extends \PHPUnit_Framework_TestCase
         );
     }
 
+    public function getLoadCatalogueTests()
+    {
+        return array(
+            array('oia', null),
+            array('oia', 'UTF-8'),
+            array('öïä', 'UTF-8'),
+            array('oia', 'ISO-8859-1'),
+            array('öïä', 'ISO-8859-1'),
+            array('цфЭ', 'UTF-8'),
+            array('цфЭ', 'KOI8-R'),
+            array('ﾖﾗﾘ', 'UTF-8'),
+            array('ﾖﾗﾘ', 'SJIS'),
+        );
+    }
+
     public function testTransChoiceFallback()
     {
         $translator = new Translator('ru', new MessageSelector());