Commit a6d9308f authored by Mike Greiling's avatar Mike Greiling

Merge branch 'leipert-proper-gettext-parsing' into 'master'

Frontend: Proper gettext extraction with gettext-extractor

Closes #48864, #42839, and #47092

See merge request gitlab-org/gitlab-ce!19378
parents 76cd1dd6 8f9442fc
......@@ -87,7 +87,7 @@ class ImporterStatus {
details = error.response.data.errors;
}
flash(__(`An error occurred while importing project: ${details}`));
flash(sprintf(__('An error occurred while importing project: %{details}'), { details }));
});
}
......
/* eslint-disable import/no-commonjs */
const SPLIT_REGEX = /\s*[\r\n]+\s*/;
/**
*
* strips newlines from strings and replaces them with a single space
*
* @example
*
* ensureSingleLine('foo \n bar') === 'foo bar'
*
* @param {String} str
* @returns {String}
*/
module.exports = function ensureSingleLine(str) {
// This guard makes the function significantly faster
if (str.includes('\n') || str.includes('\r')) {
return str
.split(SPLIT_REGEX)
.filter(s => s !== '')
.join(' ');
}
return str;
};
import Jed from 'jed';
import ensureSingleLine from './ensure_single_line';
import sprintf from './sprintf';
const languageCode = () => document.querySelector('html').getAttribute('lang') || 'en';
......@@ -10,7 +11,7 @@ delete window.translations;
@param text The text to be translated
@returns {String} The translated text
*/
const gettext = locale.gettext.bind(locale);
const gettext = text => locale.gettext.bind(locale)(ensureSingleLine(text));
/**
Translate the text with a number
......@@ -23,7 +24,10 @@ const gettext = locale.gettext.bind(locale);
@returns {String} Translated text with the number replaced (eg. '2 days')
*/
const ngettext = (text, pluralText, count) => {
const translated = locale.ngettext(text, pluralText, count).replace(/%d/g, count).split('|');
const translated = locale
.ngettext(ensureSingleLine(text), ensureSingleLine(pluralText), count)
.replace(/%d/g, count)
.split('|');
return translated[translated.length - 1];
};
......@@ -40,7 +44,7 @@ const ngettext = (text, pluralText, count) => {
@returns {String} Translated context based text
*/
const pgettext = (keyOrContext, key) => {
const normalizedKey = key ? `${keyOrContext}|${key}` : keyOrContext;
const normalizedKey = ensureSingleLine(key ? `${keyOrContext}|${key}` : keyOrContext);
const translated = gettext(normalizedKey).split('|');
return translated[translated.length - 1];
......@@ -52,8 +56,7 @@ const pgettext = (keyOrContext, key) => {
@param formatOptions for available options, please see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/DateTimeFormat
@returns {Intl.DateTimeFormat}
*/
const createDateTimeFormat =
formatOptions => Intl.DateTimeFormat(languageCode(), formatOptions);
const createDateTimeFormat = formatOptions => Intl.DateTimeFormat(languageCode(), formatOptions);
export { languageCode };
export { gettext as __ };
......
<script>
import { __ } from '~/locale';
import $ from 'jquery';
import eventHub from '../../event_hub';
......@@ -17,7 +18,7 @@ export default {
computed: {
buttonText() {
return this.isLocked ? this.__('Unlock') : this.__('Lock');
return this.isLocked ? __('Unlock') : __('Lock');
},
toggleLock() {
......
<script>
import { __ } from '~/locale';
import { __, sprintf } from '~/locale';
import Flash from '~/flash';
import tooltip from '~/vue_shared/directives/tooltip';
import issuableMixin from '~/vue_shared/mixins/issuable';
......@@ -79,11 +79,9 @@ export default {
.then(() => window.location.reload())
.catch(() =>
Flash(
this.__(
`Something went wrong trying to change the locked state of this ${
this.issuableDisplayName
}`,
),
sprintf(__('Something went wrong trying to change the locked state of this %{issuableDisplayName}'), {
issuableDisplayName: this.issuableDisplayName,
}),
),
);
},
......
require 'gettext_i18n_rails/haml_parser'
require 'gettext_i18n_rails_js/parser/javascript'
require 'json'
VUE_TRANSLATE_REGEX = /((%[\w.-]+)(?:\s))?{{ (N|n|s)?__\((.*)\) }}/
......@@ -36,6 +37,20 @@ module GettextI18nRailsJs
".vue"
].include? ::File.extname(file)
end
def collect_for(file)
gettext_messages_by_file[file] || []
end
private
def gettext_messages_by_file
@gettext_messages_by_file ||= JSON.parse(load_messages)
end
def load_messages
`node scripts/frontend/extract_gettext_all.js --all`
end
end
end
end
......
This diff is collapsed.
const argumentsParser = require('commander');
const { GettextExtractor, JsExtractors } = require('gettext-extractor');
const {
decorateJSParserWithVueSupport,
decorateExtractorWithHelpers,
} = require('gettext-extractor-vue');
const ensureSingleLine = require('../../app/assets/javascripts/locale/ensure_single_line.js');
const arguments = argumentsParser
.option('-f, --file <file>', 'Extract message from one single file')
.option('-a, --all', 'Extract message from all js/vue files')
.parse(process.argv);
const extractor = decorateExtractorWithHelpers(new GettextExtractor());
extractor.addMessageTransformFunction(ensureSingleLine);
const jsParser = extractor.createJsParser([
// Place all the possible expressions to extract here:
JsExtractors.callExpression('__', {
arguments: {
text: 0,
},
}),
JsExtractors.callExpression('n__', {
arguments: {
text: 0,
textPlural: 1,
},
}),
JsExtractors.callExpression('s__', {
arguments: {
text: 0,
},
}),
]);
const vueParser = decorateJSParserWithVueSupport(jsParser);
function printJson() {
const messages = extractor.getMessages().reduce((result, message) => {
let text = message.text;
if (message.textPlural) {
text += `\u0000${message.textPlural}`;
}
message.references.forEach(reference => {
const filename = reference.replace(/:\d+$/, '');
if (!Array.isArray(result[filename])) {
result[filename] = [];
}
result[filename].push([text, reference]);
});
return result;
}, {});
console.log(JSON.stringify(messages));
}
if (arguments.file) {
vueParser.parseFile(arguments.file).then(() => printJson());
} else if (arguments.all) {
vueParser.parseFilesGlob('{ee/app,app}/assets/javascripts/**/*.{js,vue}').then(() => printJson());
} else {
console.warn('ERROR: Please use the script correctly:');
arguments.outputHelp();
process.exit(1);
}
import ensureSingleLine from '~/locale/ensure_single_line';
describe('locale', () => {
describe('ensureSingleLine', () => {
it('should remove newlines at the start of the string', () => {
const result = 'Test';
expect(ensureSingleLine(`\n${result}`)).toBe(result);
expect(ensureSingleLine(`\t\n\t${result}`)).toBe(result);
expect(ensureSingleLine(`\r\n${result}`)).toBe(result);
expect(ensureSingleLine(`\r\n ${result}`)).toBe(result);
expect(ensureSingleLine(`\r ${result}`)).toBe(result);
expect(ensureSingleLine(` \n ${result}`)).toBe(result);
});
it('should remove newlines at the end of the string', () => {
const result = 'Test';
expect(ensureSingleLine(`${result}\n`)).toBe(result);
expect(ensureSingleLine(`${result}\t\n\t`)).toBe(result);
expect(ensureSingleLine(`${result}\r\n`)).toBe(result);
expect(ensureSingleLine(`${result}\r`)).toBe(result);
expect(ensureSingleLine(`${result} \r`)).toBe(result);
expect(ensureSingleLine(`${result} \r\n `)).toBe(result);
});
it('should replace newlines in the middle of the string with a single space', () => {
const result = 'Test';
expect(ensureSingleLine(`${result}\n${result}`)).toBe(`${result} ${result}`);
expect(ensureSingleLine(`${result}\t\n\t${result}`)).toBe(`${result} ${result}`);
expect(ensureSingleLine(`${result}\r\n${result}`)).toBe(`${result} ${result}`);
expect(ensureSingleLine(`${result}\r${result}`)).toBe(`${result} ${result}`);
expect(ensureSingleLine(`${result} \r${result}`)).toBe(`${result} ${result}`);
expect(ensureSingleLine(`${result} \r\n ${result}`)).toBe(`${result} ${result}`);
});
});
});
......@@ -1104,7 +1104,7 @@ export const collapsedSystemNotes = [
resolvable: false,
noteable_iid: 12,
note: 'changed the description',
note_html: '\n <p dir="auto">changed the description 2 times within 1 minute </p>',
note_html: ' <p dir="auto">changed the description 2 times within 1 minute </p>',
current_user: { can_edit: false, can_award_emoji: true },
resolved: false,
resolved_by: null,
......
import Vue from 'vue';
import Translate from '~/vue_shared/translate';
import Jed from 'jed';
Vue.use(Translate);
import locale from '~/locale';
import Translate from '~/vue_shared/translate';
import { trimText } from 'spec/helpers/vue_component_helper';
describe('Vue translate filter', () => {
let el;
const createTranslationMock = (key, ...translations) => {
const fakeLocale = new Jed({
domain: 'app',
locale_data: {
app: {
'': {
domain: 'app',
lang: 'vo',
plural_forms: 'nplurals=2; plural=(n != 1);',
},
[key]: translations,
},
},
});
// eslint-disable-next-line no-underscore-dangle
locale.__Rewire__('locale', fakeLocale);
};
afterEach(() => {
// eslint-disable-next-line no-underscore-dangle
locale.__ResetDependency__('locale');
});
beforeEach(() => {
Vue.use(Translate);
el = document.createElement('div');
document.body.appendChild(el);
});
it('translate single text', (done) => {
const comp = new Vue({
it('translate singular text (`__`)', done => {
const key = 'singular';
const translation = 'singular_translated';
createTranslationMock(key, translation);
const vm = new Vue({
el,
template: `
<span>
{{ __('${key}') }}
</span>
`,
}).$mount();
Vue.nextTick(() => {
expect(trimText(vm.$el.textContent)).toBe(translation);
done();
});
});
it('translate plural text (`n__`) without any substituting text', done => {
const key = 'plural';
const translationPlural = 'plural_multiple translation';
createTranslationMock(key, 'plural_singular translation', translationPlural);
const vm = new Vue({
el,
template: `
<span>
{{ __('testing') }}
{{ n__('${key}', 'plurals', 2) }}
</span>
`,
}).$mount();
Vue.nextTick(() => {
expect(
comp.$el.textContent.trim(),
).toBe('testing');
expect(trimText(vm.$el.textContent)).toBe(translationPlural);
done();
});
});
it('translate plural text with single count', (done) => {
const comp = new Vue({
describe('translate plural text (`n__`) with substituting %d', () => {
const key = '%d day';
beforeEach(() => {
createTranslationMock(key, '%d singular translated', '%d plural translated');
});
it('and n === 1', done => {
const vm = new Vue({
el,
template: `
<span>
{{ n__('${key}', '%d days', 1) }}
</span>
`,
}).$mount();
Vue.nextTick(() => {
expect(trimText(vm.$el.textContent)).toBe('1 singular translated');
done();
});
});
it('and n > 1', done => {
const vm = new Vue({
el,
template: `
<span>
{{ n__('${key}', '%d days', 2) }}
</span>
`,
}).$mount();
Vue.nextTick(() => {
expect(trimText(vm.$el.textContent)).toBe('2 plural translated');
done();
});
});
});
describe('translates text with context `s__`', () => {
const key = 'Context|Foobar';
const translation = 'Context|Foobar translated';
const expectation = 'Foobar translated';
beforeEach(() => {
createTranslationMock(key, translation);
});
it('and using two parameters', done => {
const vm = new Vue({
el,
template: `
<span>
{{ s__('Context', 'Foobar') }}
</span>
`,
}).$mount();
Vue.nextTick(() => {
expect(trimText(vm.$el.textContent)).toBe(expectation);
done();
});
});
it('and using the pipe syntax', done => {
const vm = new Vue({
el,
template: `
<span>
{{ s__('${key}') }}
</span>
`,
}).$mount();
Vue.nextTick(() => {
expect(trimText(vm.$el.textContent)).toBe(expectation);
done();
});
});
});
it('translate multi line text', done => {
const translation = 'multiline string translated';
createTranslationMock('multiline string', translation);
const vm = new Vue({
el,
template: `
<span>
{{ n__('%d day', '%d days', 1) }}
{{ __(\`
multiline
string
\`) }}
</span>
`,
}).$mount();
Vue.nextTick(() => {
expect(
comp.$el.textContent.trim(),
).toBe('1 day');
expect(trimText(vm.$el.textContent)).toBe(translation);
done();
});
});
it('translate plural text with multiple count', (done) => {
const comp = new Vue({
it('translate pluralized multi line text', done => {
const translation = 'multiline string plural';
createTranslationMock('multiline string', 'multiline string singular', translation);
const vm = new Vue({
el,
template: `
<span>
{{ n__('%d day', '%d days', 2) }}
{{ n__(
\`
multiline
string
\`,
\`
multiline
strings
\`,
2
) }}
</span>
`,
}).$mount();
Vue.nextTick(() => {
expect(
comp.$el.textContent.trim(),
).toBe('2 days');
expect(trimText(vm.$el.textContent)).toBe(translation);
done();
});
});
it('translate plural without replacing any text', (done) => {
const comp = new Vue({
it('translate pluralized multi line text with context', done => {
const translation = 'multiline string with context';
createTranslationMock('Context| multiline string', translation);
const vm = new Vue({
el,
template: `
<span>
{{ n__('day', 'days', 2) }}
{{ s__(
\`
Context|
multiline
string
\`
) }}
</span>
`,
}).$mount();
Vue.nextTick(() => {
expect(
comp.$el.textContent.trim(),
).toBe('days');
expect(trimText(vm.$el.textContent)).toBe(translation);
done();
});
......
......@@ -94,10 +94,34 @@
version "0.7.0"
resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.7.0.tgz#9a06f4f137ee84d7df0460c1fdb1135ffa6c50fd"
"@types/events@*":
version "1.2.0"
resolved "https://registry.yarnpkg.com/@types/events/-/events-1.2.0.tgz#81a6731ce4df43619e5c8c945383b3e62a89ea86"
"@types/glob@^5":
version "5.0.35"
resolved "https://registry.yarnpkg.com/@types/glob/-/glob-5.0.35.tgz#1ae151c802cece940443b5ac246925c85189f32a"
dependencies:
"@types/events" "*"
"@types/minimatch" "*"
"@types/node" "*"
"@types/jquery@^2.0.40":
version "2.0.48"
resolved "https://registry.yarnpkg.com/@types/jquery/-/jquery-2.0.48.tgz#3e90d8cde2d29015e5583017f7830cb3975b2eef"
"@types/minimatch@*":
version "3.0.3"
resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
"@types/node@*":
version "10.5.2"
resolved "https://registry.yarnpkg.com/@types/node/-/node-10.5.2.tgz#f19f05314d5421fe37e74153254201a7bf00a707"
"@types/parse5@^5":
version "5.0.0"
resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-5.0.0.tgz#9ae2106efc443d7c1e26570aa8247828c9c80f11"
"@vue/component-compiler-utils@^1.2.1":
version "1.2.1"
resolved "https://registry.yarnpkg.com/@vue/component-compiler-utils/-/component-compiler-utils-1.2.1.tgz#3d543baa75cfe5dab96e29415b78366450156ef6"
......@@ -2099,6 +2123,10 @@ css-loader@^1.0.0:
postcss-value-parser "^3.3.0"
source-list-map "^2.0.0"
css-selector-parser@^1.3:
version "1.3.0"
resolved "https://registry.yarnpkg.com/css-selector-parser/-/css-selector-parser-1.3.0.tgz#5f1ad43e2d8eefbfdc304fcd39a521664943e3eb"
css-selector-tokenizer@^0.7.0:
version "0.7.0"
resolved "https://registry.yarnpkg.com/css-selector-tokenizer/-/css-selector-tokenizer-0.7.0.tgz#e6988474ae8c953477bf5e7efecfceccd9cf4c86"
......@@ -3520,6 +3548,26 @@ getpass@^0.1.1:
dependencies:
assert-plus "^1.0.0"
gettext-extractor-vue@^4.0.1:
version "4.0.1"
resolved "https://registry.yarnpkg.com/gettext-extractor-vue/-/gettext-extractor-vue-4.0.1.tgz#69d2737eb8f1938803ffcf9317133ed59fb2372f"
dependencies:
bluebird "^3.5.1"
glob "^7.1.2"
vue-template-compiler "^2.5.0"
gettext-extractor@^3.3.2:
version "3.3.2"
resolved "https://registry.yarnpkg.com/gettext-extractor/-/gettext-extractor-3.3.2.tgz#d5172ba8d175678bd40a5abe7f908fa2a9d9473b"
dependencies:
"@types/glob" "^5"
"@types/parse5" "^5"
css-selector-parser "^1.3"
glob "5 - 7"
parse5 "^5"
pofile "^1"
typescript "^2"
glob-parent@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-3.1.0.tgz#9e6af6299d8d3bd2bd40430832bd113df906c5ae"
......@@ -3527,24 +3575,24 @@ glob-parent@^3.1.0:
is-glob "^3.1.0"
path-dirname "^1.0.0"
glob@^5.0.15:
version "5.0.15"
resolved "https://registry.yarnpkg.com/glob/-/glob-5.0.15.tgz#1bc936b9e02f4a603fcc222ecf7633d30b8b93b1"
"glob@5 - 7", glob@^7.0.3, glob@^7.0.5, glob@^7.1.1, glob@^7.1.2:
version "7.1.2"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15"
dependencies:
fs.realpath "^1.0.0"
inflight "^1.0.4"
inherits "2"
minimatch "2 || 3"
minimatch "^3.0.4"
once "^1.3.0"
path-is-absolute "^1.0.0"
glob@^7.0.3, glob@^7.0.5, glob@^7.1.1, glob@^7.1.2:
version "7.1.2"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15"
glob@^5.0.15:
version "5.0.15"
resolved "https://registry.yarnpkg.com/glob/-/glob-5.0.15.tgz#1bc936b9e02f4a603fcc222ecf7633d30b8b93b1"
dependencies:
fs.realpath "^1.0.0"
inflight "^1.0.4"
inherits "2"
minimatch "^3.0.4"
minimatch "2 || 3"
once "^1.3.0"
path-is-absolute "^1.0.0"
......@@ -5750,6 +5798,10 @@ parse-json@^2.2.0:
dependencies:
error-ex "^1.2.0"
parse5@^5:
version "5.0.0"
resolved "https://registry.yarnpkg.com/parse5/-/parse5-5.0.0.tgz#4d02710d44f3c3846197a11e205d4ef17842b81a"
parseqs@0.0.5:
version "0.0.5"
resolved "https://registry.yarnpkg.com/parseqs/-/parseqs-0.0.5.tgz#d5208a3738e46766e291ba2ea173684921a8b89d"
......@@ -5888,6 +5940,10 @@ pluralize@^7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-7.0.0.tgz#298b89df8b93b0221dbf421ad2b1b1ea23fc6777"
pofile@^1:
version "1.0.11"
resolved "https://registry.yarnpkg.com/pofile/-/pofile-1.0.11.tgz#35aff58c17491d127a07336d5522ebc9df57c954"
popper.js@^1.12.9, popper.js@^1.14.3:
version "1.14.3"
resolved "https://registry.yarnpkg.com/popper.js/-/popper.js-1.14.3.tgz#1438f98d046acf7b4d78cd502bf418ac64d4f095"
......@@ -7460,6 +7516,10 @@ typedarray@^0.0.6:
version "0.0.6"
resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"
typescript@^2:
version "2.9.2"
resolved "https://registry.yarnpkg.com/typescript/-/typescript-2.9.2.tgz#1cbf61d05d6b96269244eb6a3bce4bd914e0f00c"
uglify-es@^3.3.4:
version "3.3.9"
resolved "https://registry.yarnpkg.com/uglify-es/-/uglify-es-3.3.9.tgz#0c1c4f0700bed8dbc124cdb304d2592ca203e677"
......@@ -7756,7 +7816,7 @@ vue-style-loader@^4.1.0:
hash-sum "^1.0.2"
loader-utils "^1.0.2"
vue-template-compiler@^2.5.16:
vue-template-compiler@^2.5.0, vue-template-compiler@^2.5.16:
version "2.5.16"
resolved "https://registry.yarnpkg.com/vue-template-compiler/-/vue-template-compiler-2.5.16.tgz#93b48570e56c720cdf3f051cc15287c26fbd04cb"
dependencies:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment