i18next/i18next-scanner

Scan your code, extract translation keys/values, and merge them into i18n resource files.

i18next-scanner

Scan your code, extract translation keys/values, and merge them into i18n resource files.

Turns your code

i18n._('Loading...');
i18n._('Backslashes in single quote: \' \\ \'');
i18n._('This is \
a multiline \
string');

i18n.t('car', { context: 'blue', count: 1 }); // output: 'One blue car'
i18n.t('car', { context: 'blue', count: 2 }); // output: '2 blue cars'

<Trans i18nKey="some.key">Default text</Trans>

into resource files

{
  "Loading...": "Wird geladen...", // uses existing translation
  "Backslashes in single quote: ' \\ '": "__NOT_TRANSLATED__", // returns a custom string
  "This is a multiline string": "this is a multiline string", // returns the key as the default value
  "car": "car",
  "car_blue": "One blue car",
  "car_blue_plural": "{{count}} blue cars",
  "some": {
    "key": "Default text"
  }
}

Notice

There is a major breaking change since v1.0, and the API interface and options are not compatible with v0.x.

Checkout Migration Guide while upgrading from earlier versions.

Features

• Fully compatible with i18next - a full-featured i18n javascript library for translating your webapplication.
• Support react-i18next for parsing the Trans component
• Support Key Based Fallback to write your code without the need to maintain i18n keys. This feature is available since i18next@^2.1.0
• A standalone parser API
• A transform stream that works with both Gulp and Grunt task runner.
• Support custom transform and flush functions.

Installation

npm install --save-dev i18next-scanner

or

npm install -g i18next-scanner

Usage

CLI Usage

$ i18next-scanner

  Usage: i18next-scanner [options] <file ...>


  Options:

    -V, --version      output the version number
    --config <config>  Path to the config file (default: i18next-scanner.config.js)
    --output <path>    Path to the output directory (default: .)
    -h, --help         output usage information

  Examples:

    $ i18next-scanner --config i18next-scanner.config.js --output /path/to/output 'src/**/*.{js,jsx}'
    $ i18next-scanner --config i18next-scanner.config.js 'src/**/*.{js,jsx}'
    $ i18next-scanner '/path/to/src/app.js' '/path/to/assets/index.html'

Globbing patterns are supported for specifying file paths:

• * matches any number of characters, but not /
• ? matches a single character, but not /
• ** matches any number of characters, including /, as long as it's the only thing in a path part
• {} allows for a comma-separated list of "or" expressions
• ! at the beginning of a pattern will negate the match

Note: Globbing patterns should be wrapped in single quotes.

Examples

• examples/i18next-scanner.config.js

const fs = require('fs');
const chalk = require('chalk');

module.exports = {
    input: [
        'app/**/*.{js,jsx}',
        // Use ! to filter out files or directories
        '!app/**/*.spec.{js,jsx}',
        '!app/i18n/**',
        '!**/node_modules/**',
    ],
    output: './',
    options: {
        debug: true,
        func: {
            list: ['i18next.t', 'i18n.t'],
            extensions: ['.js', '.jsx']
        },
        trans: {
            component: 'Trans',
            i18nKey: 'i18nKey',
            defaultsKey: 'defaults',
            extensions: ['.js', '.jsx'],
            fallbackKey: function(ns, value) {
                return value;
            },

            // https://react.i18next.com/latest/trans-component#usage-with-simple-html-elements-like-less-than-br-greater-than-and-others-v10.4.0
            supportBasicHtmlNodes: true, // Enables keeping the name of simple nodes (e.g. <br/>) in translations instead of indexed keys.
            keepBasicHtmlNodesFor: ['br', 'strong', 'i', 'p'], // Which nodes are allowed to be kept in translations during defaultValue generation of <Trans>.

            // https://github.com/acornjs/acorn/tree/master/acorn#interface
            acorn: {
                ecmaVersion: 2020,
                sourceType: 'module', // defaults to 'module'
            }
        },
        lngs: ['en','de'],
        ns: [
            'locale',
            'resource'
        ],
        defaultLng: 'en',
        defaultNs: 'resource',
        defaultValue: '__STRING_NOT_TRANSLATED__',
        resource: {
            loadPath: 'i18n/{{lng}}/{{ns}}.json',
            savePath: 'i18n/{{lng}}/{{ns}}.json',
            jsonIndent: 2,
            lineEnding: '\n'
        },
        nsSeparator: false, // namespace separator
        keySeparator: false, // key separator
        interpolation: {
            prefix: '{{',
            suffix: '}}'
        },
        metadata: {},
        allowDynamicKeys: false,
    },
    transform: function customTransform(file, enc, done) {
        "use strict";
        const parser = this.parser;
        const content = fs.readFileSync(file.path, enc);
        let count = 0;

        parser.parseFuncFromString(content, { list: ['i18next._', 'i18next.__'] }, (key, options) => {
            parser.set(key, Object.assign({}, options, {
                nsSeparator: false,
                keySeparator: false
            }));
            ++count;
        });

        if (count > 0) {
            console.log(`i18next-scanner: count=${chalk.cyan(count)}, file=${chalk.yellow(JSON.stringify(file.relative))}`);
        }

        done();
    }
};

Standard API

const fs = require('fs');
const Parser = require('i18next-scanner').Parser;

const customHandler = function(key) {
    parser.set(key, '__TRANSLATION__');
};

const parser = new Parser();
let content = '';

// Parse Translation Function
// i18next.t('key');
content = fs.readFileSync('/path/to/app.js', 'utf-8');
parser
    .parseFuncFromString(content, customHandler) // pass a custom handler
    .parseFuncFromString(content, { list: ['i18next.t']}) // override `func.list`
    .parseFuncFromString(content, { list: ['i18next.t']}, customHandler)
    .parseFuncFromString(content); // use default options and handler

// Parse Trans component
content = fs.readFileSync('/path/to/app.jsx', 'utf-8');
parser
    .parseTransFromString(content, customHandler) // pass a custom handler
    .parseTransFromString(content, { component: 'Trans', i18nKey: 'i18nKey', defaultsKey: 'defaults' })
    .parseTransFromString(content, { fallbackKey: true }) // Uses defaultValue as the fallback key when the i18nKey attribute is missing
    .parseTransFromString(content); // use default options and handler

// Parse HTML Attribute
// <div data-i18n="key"></div>
content = fs.readFileSync('/path/to/index.html', 'utf-8');
parser
    .parseAttrFromString(content, customHandler) // pass a custom handler
    .parseAttrFromString(content, { list: ['data-i18n'] }) // override `attr.list`
    .parseAttrFromString(content, { list: ['data-i18n'] }, customHandler)
    .parseAttrFromString(content); // using default options and handler

console.log(parser.get());
console.log(parser.get({ sort: true }));
console.log(parser.get('translation:key', { lng: 'en'}));

Transform Stream API

The main entry function of i18next-scanner is a transform stream. You can use vinyl-fs to create a readable stream, pipe the stream through i18next-scanner to transform your code into an i18n resource object, and write to a destination folder.

Here is a simple example showing how that works:

const scanner = require('i18next-scanner');
const vfs = require('vinyl-fs');
const sort = require('gulp-sort');
const options = {
    // See options at https://github.com/i18next/i18next-scanner#options
};
vfs.src(['/path/to/src'])
    .pipe(sort()) // Sort files in stream by path
    .pipe(scanner(options))
    .pipe(vfs.dest('/path/to/dest'));

Alternatively, you can get a transform stream by calling createStream() as show below:

vfs.src(['/path/to/src'])
    .pipe(sort()) // Sort files in stream by path
    .pipe(scanner.createStream(options))
    .pipe(vfs.dest('/path/to/dest'));

Gulp

Now you are ready to set up a minimal configuration, and get started with Gulp. For example:

const gulp = require('gulp');
const sort = require('gulp-sort');
const scanner = require('i18next-scanner');

gulp.task('i18next', function() {
    return gulp.src(['src/**/*.{js,html}'])
        .pipe(sort()) // Sort files in stream by path
        .pipe(scanner({
            lngs: ['en', 'de'], // supported languages
            resource: {
                // the source path is relative to current working directory
                loadPath: 'assets/i18n/{{lng}}/{{ns}}.json',

                // the destination path is relative to your `gulp.dest()` path
                savePath: 'i18n/{{lng}}/{{ns}}.json'
            }
        }))
        .pipe(gulp.dest('assets'));
});

Grunt

Once you've finished the installation, add this line to your project's Gruntfile:

grunt.loadNpmTasks('i18next-scanner');

In your project's Gruntfile, add a section named i18next to the data object passed into grunt.initConfig(), like so:

grunt.initConfig({
    i18next: {
        dev: {
            src: 'src/**/*.{js,html}',
            dest: 'assets',
            options: {
                lngs: ['en', 'de'],
                resource: {
                    loadPath: 'assets/i18n/{{lng}}/{{ns}}.json',
                    savePath: 'i18n/{{lng}}/{{ns}}.json'
                }
            }
        }
    }
});

API

There are two ways to use i18next-scanner:

Standard API

const Parser = require('i18next-scanner').Parser;
const parser = new Parser(options);

const code = "i18next.t('key'); ...";
parser.parseFuncFromString(code);

const jsx = '<Trans i18nKey="some.key">Default text</Trans>';
parser.parseTransFromString(jsx);

const html = '<div data-i18n="key"></div>';
parser.parseAttrFromString(html);

parser.get();

parser.parseFuncFromString

Parse translation key from JS function

parser.parseFuncFromString(content)

parser.parseFuncFromString(content, { list: ['_t'] });

parser.parseFuncFromString(content, function(key, options) {
    options.defaultValue = key; // use key as the value
    parser.set(key, options);
});

parser.parseFuncFromString(content, { list: ['_t'] }, function(key, options) {
    parser.set(key, options); // use defaultValue
});

parser.parseTransFromString

Parse translation key from the Trans component

parser.parseTransFromString(content);

parser.parseTransFromString(context, { component: 'Trans', i18nKey: 'i18nKey' });

// Uses defaultValue as the fallback key when the i18nKey attribute is missing
parser.parseTransFromString(content, { fallbackKey: true });

parser.parseTransFromString(content, {
    fallbackKey: function(ns, value) {
        // Returns a hash value as the fallback key
        return sha1(value);
    }
});

parser.parseTransFromString(content, function(key, options) {
    options.defaultValue = key; // use key as the value
    parser.set(key, options);
});

parser.parseAttrFromString

Parse translation key from HTML attribute

parser.parseAttrFromString(content)

parser.parseAttrFromString(content, { list: ['data-i18n'] });

parser.parseAttrFromString(content, function(key) {
    const defaultValue = key; // use key as the value
    parser.set(key, defaultValue);
});

parser.parseAttrFromString(content, { list: ['data-i18n'] }, function(key) {
    parser.set(key); // use defaultValue
});

parser.get

Get the value of a translation key or the whole i18n resource store

// Returns the whole i18n resource store
parser.get();

// Returns the resource store with the top-level keys sorted by alphabetical order
parser.get({ sort: true });

// Returns a value in fallback language (@see options.fallbackLng) with namespace and key
parser.get('ns:key');

// Returns a value with namespace, key, and lng
parser.get('ns:key', { lng: 'en' });

parser.set

Set a translation key with an optional defaultValue to i18n resource store

// Set a translation key
parser.set(key);

// Set a translation key with default value
parser.set(key, defaultValue);

// Set a translation key with default value using options
parser.set(key, {
    defaultValue: defaultValue
});

Transform Stream API

const scanner = require('i18next-scanner');
scanner.createStream(options, customTransform /* optional */, customFlush /* optional */);

customTransform

The optional customTransform function is provided as the 2nd argument for the transform stream API. It must have the following signature: function (file, encoding, done) {}. A minimal implementation should call the done() function to indicate that the transformation is done, even if that transformation means discarding the file. For example:

const scanner = require('i18next-scanner');
const vfs = require('vinyl-fs');
const customTransform = function _transform(file, enc, done) {
    const parser = this.parser;
    const content = fs.readFileSync(file.path, enc);

    // add your code
    done();
};

vfs.src(['/path/to/src'])
    .pipe(scanner(options, customTransform))
    .pipe(vfs.dest('path/to/dest'));

To parse a translation key, call parser.set(key, defaultValue) to assign the key with an optional defaultValue. For example:

const customTransform = function _transform(file, enc, done) {
    const parser = this.parser;
    const content = fs.readFileSync(file.path, enc);

    parser.parseFuncFromString(content, { list: ['i18n.t'] }, function(key) {
        const defaultValue = '__L10N__';
        parser.set(key, defaultValue);
    });

    done();
};

Alternatively, you may call parser.set(defaultKey, value) to assign the value with a default key. The defaultKey should be unique string and can never be null, undefined, or empty. For example:

const hash = require('sha1');
const customTransform = function _transform(file, enc, done) {
    const parser = this.parser;
    const content = fs.readFileSync(file.path, enc);

    parser.parseFuncFromString(content, { list: ['i18n._'] }, function(key) {
        const value = key;
        const defaultKey = hash(value);
        parser.set(defaultKey, value);
    });

    done();
};

customFlush

The optional customFlush function is provided as the last argument for the transform stream API, it is called just prior to the stream ending. You can implement your customFlush function to override the default flush function. When everything's done, call the done() function to indicate the stream is finished. For example:

const scanner = require('i18next-scanner');
const vfs = require('vinyl-fs');
const customFlush = function _flush(done) {
    const parser = this.parser;
    const resStore = parser.getResourceStore();

    // loop over the resStore
    Object.keys(resStore).forEach(function(lng) {
        const namespaces = resStore[lng];
        Object.keys(namespaces).forEach(function(ns) {
            const obj = namespaces[ns];
            // add your code
        });
    });

    done();
};

vfs.src(['/path/to/src'])
    .pipe(scanner(options, customTransform, customFlush))
    .pipe(vfs.dest('/path/to/dest'));

Default Options

Below are the configuration options with their default values:

{
    compatibilityJSON: 'v3', // One of: 'v1', 'v2', 'v3', 'v4
    debug: false,
    removeUnusedKeys: false,
    sort: false,
    attr: {
        list: ['data-i18n'],
        extensions: ['.html', '.htm'],
    },
    func: {
        list: ['i18next.t', 'i18n.t'],
        extensions: ['.js', '.jsx'],
    },
    trans: {
        component: 'Trans',
        i18nKey: 'i18nKey',
        defaultsKey: 'defaults',
        extensions: ['.js', '.jsx'],
        fallbackKey: false,

        // https://react.i18next.com/latest/trans-component#usage-with-simple-html-elements-like-less-than-br-greater-than-and-others-v10.4.0
        supportBasicHtmlNodes: true, // Enables keeping the name of simple nodes (e.g. <br/>) in translations instead of indexed keys.
        keepBasicHtmlNodesFor: ['br', 'strong', 'i', 'p'], // Which nodes are allowed to be kept in translations during defaultValue generation of <Trans>.

        // https://github.com/acornjs/acorn/tree/master/acorn#interface
        acorn: {
            ecmaVersion: 2020,
            sourceType: 'module', // defaults to 'module'
        },
    },
    lngs: ['en'],
    ns: ['translation'],
    defaultLng: 'en',
    defaultNs: 'translation',
    defaultValue: '',
    resource: {
        loadPath: 'i18n/{{lng}}/{{ns}}.json',
        savePath: 'i18n/{{lng}}/{{ns}}.json',
        jsonIndent: 2,
        lineEnding: '\n',
    },
    nsSeparator: ':',
    keySeparator: '.',
    pluralSeparator: '_',
    contextSeparator: '_',
    contextDefaultValues: [],
    interpolation: {
        prefix: '{{',
        suffix: '}}',
    },
    metadata: {},
    allowDynamicKeys: false,
}

compatibilityJSON

Type: String Default: 'v3'

The compatibilityJSON version to use for plural suffixes.

See https://www.i18next.com/misc/json-format for details.

debug

Type: Boolean Default: false

Set to true to turn on debug output.

removeUnusedKeys

Type: Boolean Default: false

Set to true to remove unused translation keys from i18n resource files.

sort

Type: Boolean Default: false

Set to true if you want to sort translation keys in ascending order.

attr

Type: Object or false

If an Object is supplied, you can either specify a list of attributes and extensions, or override the default.

{ // Default
    attr: {
        list: ['data-i18n'],
        extensions: ['.html', '.htm']
    }
}

You can set attr to false to disable parsing attribute as below:

{
    attr: false
}

func

Type: Object or false

If an Object is supplied, you can either specify a list of translation functions and extensions, or override the default.

{ // Default
    func: {
        list: ['i18next.t', 'i18n.t'],
        extensions: ['.js', '.jsx']
    }
}

You can set func to false to disable parsing translation function as below:

{
    func: false
}

trans

Type: Object or false

If an Object is supplied, you can specify a list of extensions, or override the default.

{ // Default
    trans: {
        component: 'Trans',
        i18nKey: 'i18nKey',
        defaultsKey: 'defaults',
        extensions: ['.js', '.jsx'],
        fallbackKey: false,

        // https://react.i18next.com/latest/trans-component#usage-with-simple-html-elements-like-less-than-br-greater-than-and-others-v10.4.0
        supportBasicHtmlNodes: true, // Enables keeping the name of simple nodes (e.g. <br/>) in translations instead of indexed keys.
        keepBasicHtmlNodesFor: ['br', 'strong', 'i', 'p'], // Which nodes are allowed to be kept in translations during defaultValue generation of <Trans>.

        // https://github.com/acornjs/acorn/tree/master/acorn#interface
        acorn: {
            ecmaVersion: 2020,
            sourceType: 'module', // defaults to 'module'
        },
    }
}

You can set trans to false to disable parsing Trans component as below:

{
    trans: false
}

The fallbackKey can either be a boolean value, or a function like so:

fallbackKey: function(ns, value) {
    // Returns a hash value as the fallback key
    return sha1(value);
}

You can pass RexExp to trans.component in case you want to match multiple things:

component: /Trans$/

lngs

Type: Array Default: ['en']

An array of supported languages.

ns

Type: String or Array Default: ['translation']

A namespace string or an array of namespaces.

defaultLng

Type: String Default: 'en'

The default language used for checking default values.

defaultNs

Type: String Default: 'translation'

The default namespace used if not passed to translation function.

defaultValue

Type: String or Function Default: ''

The default value used if not passed to parser.set.

Examples

Provides the default value with a string:

{
    defaultValue: '__NOT_TRANSLATED__'
}

Provides the default value as a callback function:

{
    // @param {string} lng The language currently used.
    // @param {string} ns The namespace currently used.
    // @param {string} key The translation key.
    // @return {string} Returns a default value for the translation key.
    defaultValue: function(lng, ns, key) {
        if (lng === 'en') {
            // Return key as the default value for English language
            return key;
        }
        // Return the string '__NOT_TRANSLATED__' for other languages
        return '__NOT_TRANSLATED__';
    }
}

resource

Type: Object

Resource options:

{ // Default
    resource: {
        // The path where resources get loaded from. Relative to current working directory.
        loadPath: 'i18n/{{lng}}/{{ns}}.json',

        // The path to store resources. Relative to the path specified by `gulp.dest(path)`.
        savePath: 'i18n/{{lng}}/{{ns}}.json',

        // Specify the number of space characters to use as white space to insert into the output JSON string for readability purpose.
        jsonIndent: 2,

        // Normalize line endings to '\r\n', '\r', '\n', or 'auto' for the current operating system. Defaults to '\n'.
        // Aliases: 'CRLF', 'CR', 'LF', 'crlf', 'cr', 'lf'
        lineEnding: '\n'
    }
}

loadPath and savePath can be both be defined as Function with parameters lng and ns

{ // Default
    resource: {
        // The path where resources get loaded from. Relative to current working directory.
        loadPath: function(lng, ns) {
            return 'i18n/'+lng+'/'+ns+'.json';
        },

        // The path to store resources. Relative to the path specified by `gulp.dest(path)`.
        savePath: function(lng, ns) {
            return 'i18n/'+lng+'/'+ns+'.json';
        },

        // Specify the number of space characters to use as white space to insert into the output JSON string for readability purpose.
        jsonIndent: 2,

        // Normalize line endings to '\r\n', '\r', '\n', or 'auto' for the current operating system. Defaults to '\n'.
        // Aliases: 'CRLF', 'CR', 'LF', 'crlf', 'cr', 'lf'
        lineEnding: '\n'
    }
}

keySeparator

Type: String or false Default: '.'

Key separator used in translation keys.

Set to false to disable key separator if you prefer having keys as the fallback for translation (e.g. gettext). This feature is supported by [email protected]. Also see Key based fallback at https://www.i18next.com/principles/fallback#key-fallback.

nsSeparator

Type: String or false Default: ':'

Namespace separator used in translation keys.

Set to false to disable namespace separator if you prefer having keys as the fallback for translation (e.g. gettext). This feature is supported by [email protected]. Also see Key based fallback at https://www.i18next.com/principles/fallback#key-fallback.

context

Type: Boolean or Function Default: true

Whether to add context form key.

context: function(lng, ns, key, options) {
    return true;
}

contextFallback

Type: Boolean Default: true

Whether to add a fallback key as well as the context form key.

contextSeparator

Type: String Default: '_'

The character to split context from key.

contextDefaultValues

Type: Array Default: []

A list of default context values, used when the scanner encounters dynamic value as a context. For a list of ['male', 'female'] the scanner will generate an entry for each value.

plural

Type: Boolean or Function Default: true

Whether to add plural form key.

plural: function(lng, ns, key, options) {
    return true;
}

pluralFallback

Type: Boolean Default: true

Whether to add a fallback key as well as the plural form key.

pluralSeparator

Type: String Default: '_'

The character to split plural from key.

interpolation

Type: Object

interpolation options

{ // Default
    interpolation: {
        // The prefix for variables
        prefix: '{{',

        // The suffix for variables
        suffix: '}}'
    }
}

metadata

Type: Object Default: {}

This can be used to pass any additional information regarding the string.

allowDynamicKeys

Type: Boolean Default: false

This can be used to allow dynamic keys e.g. friend${DynamicValue}

Example Usage:

  transform: function customTransform(file, enc, done) {
    'use strict';
    const parser = this.parser;

    const contexts = {
      compact: ['compact'],
      max: ['Max'],
    };

    const keys = {
        difficulty: { list: ['Normal', 'Hard'] },
        minMax: { list: ['Min', 'Max'] },
    };

    const content = fs.readFileSync(file.path, enc);

    parser.parseFuncFromString(content, { list: ['i18next.t', 'i18n.t'] }, (key, options) => {
      // Add context based on metadata
      if (options.metadata?.context) {
        delete options.context;
        const context = contexts[options.metadata?.context];
        parser.set(key, options);
        for (let i = 0; i < context?.length; i++) {
          parser.set(`${key}${parser.options.contextSeparator}${context[i]}`, options);
        }
      }

      // Add keys based on metadata (dynamic or otherwise)
      if (options.metadata?.keys) {
        const list = keys[options.metadata?.keys].list;
        for (let i = 0; i < list?.length; i++) {
          parser.set(`${key}${list[i]}`, options);
        }
      }

      // Add all other non-metadata related keys
      if (!options.metadata) {
        parser.set(key, options);
      }
    });

    done();

Integration Guide

Checkout Integration Guide to learn how to integrate with React, Gettext Style I18n, and Handlebars.

License

MIT

i18next-scanner's Issues

React class parse error

I am using i18next-scanner for get strings from my components. But if I use class field like this

export default class Main extends Component {
static testvalue = 12;
render(){return(<div><Trans>some string</Trans></div>)}
}

"static testvalue = 12" cause this error

18next-scanner: Unable to parse Trans component with the content
{ SyntaxError: Unexpected token (10:19)

If I don't use this way for define class fields all works good. How can I fix it?

Support react-i18next "defaults" prop

Version

• i18next: 11.2.2
• i18next-scanner: 2.4.6
• react-i18next: 7.7.0

Configuration

options: {
	// use strings as keys
	nsSeparator: false,
	keySeparator: false,
	// settings
	defaultNs: "frontend",
	lngs: ["en"],
	resource: {
		loadPath: "{{lng}}/{{ns}}.json",
		savePath: "{{lng}}/{{ns}}.json",
	},
	func: false,
	trans: false,
	defaultValue: (language, namespace, key) => key,
},

react-i18n v7.7.0 introduced the defaults prop (see i18next/react-i18next#439). I expect the following code:

<Trans
  defaults="Hello <0>{val}</0>!"
  tOptions={{val: "World"}}
  components=[<strong>foo</strong>]
/>

to lead to the following output:

{
  "Hello <0>{val}</0>!": "Hello <0>{val}</0>!"
}

Error when comparing file extensions in transform function

Hi,

I think there is an error in the transform function when comparing file extensions with extentions from options. extname should be compared against options.attr.extensions and options.func.extensions instead of options.attr.list and options.func.list.

    if (_lodash2.default.isObject(options.attr) && _lodash2.default.includes(options.attr.list, extname)) {
        // Parse attribute (e.g. data-i18n="key")
        parser.parseAttrFromString(content);
    }

    if (_lodash2.default.isObject(options.func) && _lodash2.default.includes(options.func.list, extname)) {
        // Parse translation function (e.g. i18next.t('key'))
        parser.parseFuncFromString(content);
    }

should probably be

    if (_lodash2.default.isObject(options.attr) && _lodash2.default.includes(options.attr.extensions, extname)) {
        // Parse attribute (e.g. data-i18n="key")
        parser.parseAttrFromString(content);
    }

    if (_lodash2.default.isObject(options.func) && _lodash2.default.includes(options.func.extensions, extname)) {
        // Parse translation function (e.g. i18next.t('key'))
        parser.parseFuncFromString(content);
    }

Not all files generated when using with grunt.

I have an issue trying to use i18next-scanner with grunt as it looks like the task is finishing before all files from vfs are written in the disk.

Merge projects?

I'm the author of i18next-parser. When you created the project github auto subscribed me to the notifications and I thought that I'd say hi.

Do you want to collaborate rather than maintaining two similar projects?

Link element broken in Trans processing

Our use of parse5 to parse Trans element results in some interesting behaviour: parse5 enforces HTML rules, which can lead to unexpected results. See inikulin/parse5#219 for details.

The net result is that this input:

 <Trans i18nKey="custom.label">
  I agree with the <Link to="/terms-and-conditions">terms</Link>.
</Trans>

becomes I agree to the <1></1>terms. – definitely not the desired result.

@cheton I'll try to deal with this one - I will need a fix in the next few days anyway.

Extraction of translation values

I am using this plug-in for the extraction of translation key and it's corresponding value from html files. After executing my gulp task, translation keys are extracted but with blank (" ") values. Am I missing any configuration??

Html text:
<h2 data-i18n="home.DefaultKey">Some Text</h2>

Gulp task:

var gulp = require('gulp');
var i18next = require('i18next-scanner');

gulp.task('i18nextScan', function () {
    return gulp.src('src/**/*.html')
        .pipe(i18next({
            // a list of supported languages
            lngs: ['en'],

            // the source path is relative to current working directory
            resGetPath: 'locale/__lng__/__ns__.json',

            // the destination path is relative to your `gulp.dest()` path
            resSetPath: '__lng__/__ns__.json'
        }))
        .pipe(gulp.dest('locale'));
});

Output generated:
path: locale/en/translation.json

{
    "home": {
        "DefaultKey": ""
      }
}

How should I configure the scanner to make it works with react-i18next

I want to use react-i18next with i18next-scanner but I can't find how to configure it.

I have try the following:

Version

• i18next: 3.4.1
• i18next-scanner: 1.4.1

Configuration

  var scanner = require('i18next-scanner');
  var vfs = require('vinyl-fs');
  var sort = require('gulp-sort');
  var options = {
    list: ['this.pros.t', 'props.t', 't', 'i18next.t']
  };
  vfs.src(['src/*.js'])
      .pipe(sort()) // Sort files in stream by path
      .pipe(scanner(options))
      .pipe(vfs.dest('.'));

But this do not extract the any strings from the react-i18next example project.

This should work with the example, right ?

We can the following use of the i18next in react:

<a>{this.props.t('nav:link1')}</a>
<a>{props.t('nav:link1')}</a>
<a>{t('nav:link1')}</a>

Extraction breaks for whole file when using shorthand JSX fragments

When attempting to parse a file on 2.6.2 that contains a shorthand JSX fragment (<>like this</>), the scanner fails to parse the whole file and doesn't include any string from it.

While we can replace the fragments in our code base with (<React.Fragment>like this</React.Fragment>), we'd like to be able to use them. The biggest problem is that the existance of one shorthand fragment breaks a whole file, not just the Trans components using it/inside it.

From a cursory look it seems like acorn-jsx-walk hasn't been updated since 2016, so it may be using some old babel or custom code under the hood that is unable to cope with shorthand fragments. When running i18next-scanner in debug mode, I get the following message:

{ SyntaxError: Unexpected token (3:38)
    at Parser.pp.raise (/opt/source/node_modules/acorn-jsx-walk/node_modules/acorn/dist/acorn.js:943:13)
    at Parser.pp.unexpected (/opt/source/node_modules/acorn-jsx-walk/node_modules/acorn/dist/acorn.js:1503:8)
    at Parser.pp.expect (/opt/source/node_modules/acorn-jsx-walk/node_modules/acorn/dist/acorn.js:1497:26)
    at Parser.pp.parseBindingList (/opt/source/node_modules/acorn-jsx-walk/node_modules/acorn/dist/acorn.js:1149:40)
    at Parser.pp.parseFunctionParams (/opt/source/node_modules/acorn-jsx-walk/node_modules/acorn/dist/acorn.js:2071:22)
    at Parser.pp.parseFunction (/opt/source/node_modules/acorn-jsx-walk/node_modules/acorn/dist/acorn.js:2064:8)
    at Parser.pp.parseFunctionStatement (/opt/source/node_modules/acorn-jsx-walk/node_modules/acorn/dist/acorn.js:1830:15)
    at Parser.pp.parseStatement (/opt/source/node_modules/acorn-jsx-walk/node_modules/acorn/dist/acorn.js:1709:19)
    at Parser.pp.parseExport (/opt/source/node_modules/acorn-jsx-walk/node_modules/acorn/dist/acorn.js:2169:29)
    at Parser.pp.parseStatement (/opt/source/node_modules/acorn-jsx-walk/node_modules/acorn/dist/acorn.js:1741:85) pos: 91, loc: Position { line: 3, column: 38 }, raisedAt: 92 }

Writing a test should be trivial.

Version

• i18next: 11.3.3
• i18next-scanner: 2.6.2

Configuration

const fs = require("fs");

module.exports = {
	options: {
		// use strings as keys
		nsSeparator: false,
		keySeparator: false,
		// settings
		defaultNs: "frontend",
		lngs: ["en"],
		resource: {
			loadPath: "{{lng}}/{{ns}}.json",
			savePath: "{{lng}}/{{ns}}.json",
		},
		func: false,
		trans: false,
		defaultValue: (language, namespace, key) => key,
	},
	transform(file, enc, done) {
		const parser = this.parser;
		const content = fs.readFileSync(file.path, enc);

		parser.parseFuncFromString(
			content,
			{
				list: ["t"],
			},
			(key, options) => {
				if (["some", "blacklisted", "keys"].indexOf(key) !== -1) {
					return;
				}
				parser.set(key, options);
			},
		);

		parser.parseTransFromString(
			content,
			{
				component: ["Trans"],
				i18nKey: false,
			},
			(key, options) => {
				parser.set(options.defaultValue, options);
			},
		);

		done();
	},
};

Is it possible to pluralise the collected strings automatically from <Trans> components ?

Version

• i18next: 11.5.0
• i18next-scanner: 2.6.5

Configuration

const fs = require('fs');
const chalk = require('chalk');

module.exports = {
  options: {
    debug: true,
    removeUnusedKeys: true,
    func: {
      list: ['i18next.t', 'i18n.t', 't'],
      extensions: ['.js']
    },
    trans: {
      component: 'Trans',
      i18nKey: 'i18nKey',
      defaultsKey: 'defaults',
      extensions: ['.js'],
      fallbackKey: (ns, value) => value
    },
    lngs: ['en'],
    ns: ['translation'],
    defaultLng: 'en',
    defaultNs: 'translation',
    fallbackLng: 'en',
    defaultValue: '__STRING_NOT_TRANSLATED__',
    resource: {
      loadPath: 'src/i18n/{{lng}}/{{ns}}.json',
      savePath: 'src/i18n/{{lng}}/{{ns}}.json',
      jsonIndent: 2,
      lineEnding: '\n'
    },
    nsSeparator: false, // namespace separator
    keySeparator: false, // key separator
    interpolation: {
      prefix: '{{',
      suffix: '}}'
    }
  },
  transform: function customTransform(file, enc, done) {
    const parser = this.parser;
    const content = fs.readFileSync(file.path, enc);
    let count = 0;

    parser.parseFuncFromString(content, { list: ['i18next._', 'i18next.__'] }, (key, options) => {
      parser.set(key, Object.assign({}, options, {
        nsSeparator: false,
        keySeparator: false
      }));
      ++count;
    });
    if (count > 0) {
      console.log(`i18next-scanner: count=${chalk.cyan(count)}, file=${chalk.yellow(JSON.stringify(file.relative))}`);
    }
    done();
  }
};

<Trans count={count}>
    you have {{ count }}
</Trans>

and be collected as the following

  "unreadMessages": "you have <1>{{count}}</1> unread message.",
  "unreadMessages_plural": "you have <1>{{count}}</1> unread messages."

How I can remove old translations which was already deleted from code?

I already try to use loadPath only as separate file. My script:

const scanner = require('i18next-scanner');
const languages = require('./languages.json');
const vfs = require('vinyl-fs');
const fs = require('fs');
const path = require('path');

Object.keys(languages).forEach((k) => {
  const fromFile = path.resolve(__dirname, `../locales/${languages[k]}/translation.json`);
  const toFile = path.resolve(__dirname, `../locales/${languages[k]}/translation-prev.json`);
  fs.renameSync(fromFile, toFile);
});

const options = {
  attr: false,
  func: {
    list: ['_', 'trans'],
    extensions: ['.js', '.jsx', '.php', '.twig'],
  },
  lngs: Object.keys(languages).map(k => languages[k]),
  ns: 'translation',
  defaultNs: 'translation',
  resource: {
    loadPath: 'locales/{{lng}}/{{ns}}-prev.json',
    savePath: '{{lng}}/{{ns}}.json',
  },
  nsSeparator: false,
  keySeparator: false,
};

vfs.src([
  'source/js/**/*.js'
])
.pipe(scanner(options))
.pipe(vfs.dest('locales'));

Parsing `<template>` html files is not working.

Version

• i18next: 9.1.0
• i18next-scanner: 2.4.4

---

Html walk function is not working for people who wants to scan html like <template></template>.

i18next-scanner/src/parser.js

Lines 494 to 508 in 9d458a1

	const walk = (nodes) => {
	nodes.forEach(node => {
	if (node.attrs) {
	node.attrs.forEach(attr => {
	if (attrs.indexOf(attr.name) !== -1) {
	const values = attr.value.split(';');
	values.forEach(parseAttributeValue);
	}
	});
	}
	if (node.childNodes) {
	walk(node.childNodes);
	}
	});
	};

`customHandler` is not called for `parseAttrFromString()`

Version

• i18next: 9.1.0
• i18next-scanner: 2.4.4

---

As you can see customHandler is never used in parseAttrFromString().

i18next-scanner/src/parser.js

Lines 457 to 513 in 9d458a1

	// Parses translation keys from `data-i18n` attribute in HTML
	// <div data-i18n="[attr]ns:foo.bar;[attr]ns:foo.baz">
	// </div>
	parseAttrFromString(content, opts = {}, customHandler = null) {
	let setter = this.set.bind(this);
	if (_.isFunction(opts)) {
	setter = opts;
	opts = {};
	}
	const attrs = (opts.list !== undefined)
	? ensureArray(opts.list)
	: ensureArray(this.options.attr.list);
	if (attrs.length === 0) {
	return this;
	}
	const ast = parse5.parse(content);
	const parseAttributeValue = (key) => {
	key = _.trim(key);
	if (key.length === 0) {
	return;
	}
	if (key.indexOf('[') === 0) {
	const parts = key.split(']');
	key = parts[1];
	}
	if (key.indexOf(';') === (key.length - 1)) {
	key = key.substr(0, key.length - 2);
	}
	setter(key);
	};
	const walk = (nodes) => {
	nodes.forEach(node => {
	if (node.attrs) {
	node.attrs.forEach(attr => {
	if (attrs.indexOf(attr.name) !== -1) {
	const values = attr.value.split(';');
	values.forEach(parseAttributeValue);
	}
	});
	}
	if (node.childNodes) {
	walk(node.childNodes);
	}
	});
	};
	walk(ast.childNodes);
	return this;
	}

Any chance for a CLI?

We're using CRA which has neither Gulp nor Grunt, and hides webpack config (unless you eject). It'd be great to have a simple CLI that can be run without other dependencies.

Singular and Plural

Is there a workaround to support Plurals?: http://i18next.com/translate/pluralSimple/

<Trans> component in Typescript files is no longer parsed

This is most probably caused by dfd5db1 (2.6.2)

Version

• i18next: 11.3.6
• i18next-scanner: 2.6.3

Configuration

{
  options: {
    debug: "true",
    lngs: ["en"],
    attr: false,
    func: {
      extensions: [".ts", ".tsx"],
      list: ["t", "props.t", "i18n.t"],
    },
    trans: {
      extensions: [".ts", ".tsx"],
      fallbackKey: true,
    },
    keySeparator: false,
    nsSeparator: false,
  },
}

Sample file

// test.tsx

import * as React from "react"
import { Trans } from "react-i18next"

const TestComp: React.SFC = () => (
  <div className="test-comp">
    <h3>
      <Trans>This is a test</Trans>
    </h3>
  </div>
)

Result

"This is a test" is not present in the output. The strings using the t() helper function are still correctly extracted.

parser returns commented <Trans/> results

Edgecase:
When is commented, the parser still returns it.

// WORKAROUND: There seems to be a bug in [email protected] that introduces invalid keys for the
// message's <Trans> when we break the lines between the </p><p>. Breaking lines at a normal space seems to work
message: (
   <Trans>...</Trans>

Version

• i18next-scanner: 2.4.4

Add backquote support

Version

• i18next: 10.0.7
• i18next-scanner: 2.1.2

Configuration

{ // i18next-scanner options
    defaultValue: '__STRING_NOT_TRANSLATED__',
    resource: {
      loadPath: 'i18n/{{lng}}/{{ns}}.json',
      savePath: 'i18n/{{lng}}/{{ns}}.json',
      jsonIndent: 2
    },
    nsSeparator: false,
    keySeparator: false
}

For multiline string, I sometimes use ES6 template strings and it seems i18next-scanner does not support backquote according to the test case.

My test case is:

i18next.t(`aString`);

or even further

i18next.t(`line1
  line2
  line3`);

Could you please help to add the backquote parsing support?
Thank you.

should not escape backslash

Version

• i18next: 4.2.0
• i18next-scanner: 1.5.1

Configuration

{ // i18next-scanner options
        lngs: ['zh-CN'],
        func: {
          list: ['i18next.t', 'i18n.t', 't'],
          extensions: ['.js', '.jsx']
        },
        resource: {
          savePath: `{{lng}}/${moduleName}.json`
        },
        keySeparator: false,
        nsSeparator: false
}

I'm using Key Based Fallback. If I write code like i18next.t('first line\nsecond line'), i18next-scanner will generate json like this:

{
   "first line\\nsecond line": ""
}

The double backslash which escapes the original new-line character will cause the key in json file different with the original key, leading to invalid translation.

i18next-scanner overwrites all my translation values

Whenever I run gulp i18next, I get overwritten translations. Keys are scanned well, but all the values are overwritten. What am I doing wrong? I just want new keys added.

When I run it in debug mode, terminal says: i18next-scanner: Added a new translation key { "hello_title": "" } to ".locales/en/translations.json".

I tried adding i18next-scanner.config.js file with the config you specified in the tutorial, but no luck. What am I missing?

Thanks in advance!

Version

• i18next: 11.3.2
• react-i18next: 7.6.1
• i18next-scanner: 2.4.6

Configuration

My gulp file:

var gulp = require('gulp');
const sort = require('gulp-sort');
const scanner = require('i18next-scanner');

gulp.task('i18next', function() {
    return gulp.src(['src/**/*.{js,html}'])
        .pipe(sort()) // Sort files in stream by path
        .pipe(scanner({
            lngs: ['en', 'de'], // supported languages
            resource: {
                // the source path is relative to current working directory
                loadPath: './locales/{{lng}}/translations.json',
                
                // the destination path is relative to your `gulp.dest()` path
                savePath: './dists/locales/{{lng}}/translations.json'
						},
						func: {
							list: ['t']
						},
						debug: true,
						config: 'i18next-scanner.config.js'
        }))
        .pipe(gulp.dest('public'));
});

My jsx looks like this:

<h2>{t('hello_title')}</h2>

Disabiling nsSeparator breaks the defaultValue fallback

When nsSeparator is false, translation will not fallback to defaultValue:

https://github.com/i18next/i18next-scanner/blob/master/src/parser.js#L367

If key is missing the defaultValue should take place even without nsSeparator.

Thanks.

Esprima used only when code ends with comma

Version

• i18next-scanner: master

I am curious why you guys use a fragile regexp to parse function calls and only resort to esprima when string ends with comma ...?

Not keeping already written values

Hi guys,

Version

• i18next: 11.3.6
• i18next-scanner: 2.6.3
• Using gulp execution

Configuration

{ // i18next-scanner options
    lngs: ['en', 'fr'],
    defaultLng: 'en',
    sort: true,
    attr: {
        list: ['ng-i18next'],
        extensions: ['.html', '.htm']
    },
    func: {
        list: ['i18next.t', 'i18n.t'],
        extensions: ['.js']
    },
    trans: {
      extensions: [],
    },
    ns: [
      'buttons',
      'errors',
      'export',
      'files',
      'home',
      'modal',
    ],
    defaultValue: '_STRING_NOT_TRANSLATED_',
    resource: {
      loadPath: '{{lng}}/{{ns}}.json',
      savePath: '{{lng}}/{{ns}}.json',
    },
  };
}

I have already done some translations but when I relaunch the scanner it will erase all my translation to set them again with the default value.

In the parser.js l.286, is trying to load the resource file but it cannot find it because it is not using the right path.
If I set an absolute path in the config options like below it works fine :

  loadPath: path.resolve('./src/locales/{{lng}}/{{ns}}.json'),

But it sounds strange to have to use absolute path for the loadPath and not for the savePath. Is there something that I am doing wrong to make it work ?

Thanks for you work

[question] about log "Translation key "myKey" has multiple different default values. Using first default"

When I generate two translation files (en and de) after modifying the "de" file, I have this log.

en-translations.json :

{  
  "To get started, edit <1>src/App.js</1> and save to reload.": "To get started, edit <1>src/App.js</1> and save to reload.",
  "Welcome to React": "Welcome to React"
}

de-translations.json :

{
  "To get started, edit <1>src/App.js</1> and save to reload.": "Starte in dem du, <1>src/App.js</1> editierst und speicherst.",
  "Welcome to React": "Willkommen bei React und react-i18next"
}

console :
i18next-scanner: Translation key "To get started, edit <1>src/App.js</1> and save to reload." has multiple different default values. Using first default

Can you explain to me why this happen with tag and not with t() function ?

Thanks for your help.

Incorrect whitespace handling in JSX

Version

• i18next: 10.3.0
• i18next-scanner: 2.4.4
• react-i18next: 7.3.6

Hey there! Thanks for your great work on this tool - it's super useful. I've noticed a few places where it doesn't work as expected around whitespace in JSX:

A few examples:

  <Trans>
    hello
    <span>world</span>
  </Trans>

Expected key (generated by react-i18next): hello<1>world</1>
Actual key (generated by i18next-scanner): hello <1>world</1>

  <Trans>
    <span>hello</span>
    world
  </Trans>

Expected key (generated by react-i18next): <0>hello</0>world
Actual key (generated by i18next-scanner): <0>hello</0> world

This is an area where JSX behaves a bit differently to HTML. To keep a space between the two words, it's quite common to add {' '}:

  <Trans>
    hello{' '}
    <span>world</span>
  </Trans>

Expected key (generated by react-i18next): hello <2>world</2>
Actual key (generated by i18next-scanner): hello<1>{' '}</1> <3>world</3>

Can't parse function with $ dollar sign

Hello,

Version

• i18next: 11.3.2
• i18next-scanner: 2.4.6

Configuration

const Parser = require('i18next-scanner').Parser;
const parser = new Parser();

const code = "i18next.t('Hello world'); ...";
parser.parseFuncFromString(code, {list: ['i18next.t']});

console.log(parser.get());

Outputs :

{ en: { translation: { 'Hello world': '' } } }

const Parser = require('i18next-scanner').Parser;
const parser = new Parser();

const code = "$t('Hello world'); ...";
parser.parseFuncFromString(code, {list: ['$t']});

console.log(parser.get());

Outputs :

{ en: { translation: {} } }

Is it a bug ?
I have also tried to escape \$t, doesn't work.

Regards
Thanks

[i18next-scanner v2.6] Cannot extract translation from component props

Cannot extract translation from component props with i18next-scanner v2.6:

<Component render={<Trans>translation from props</Trans>} />

It's probably a missing test case when parsing JSX element attributes.

Broken npm install

Hi. Yesterday I have faced an issue when trying to make npm install on a project, which has i18next-scanner.

npm debug logs didn't give me much information, but by removing all the dependencies from my project and then trying to install each individually I have narrowed the problem to be with this particular library.

The error I'm facing looks like this:

This might be a problem with one of the dependencies of this project or even with a dependency of their dependencies, however, all other libraries install without any error.

Multiline strings not parsed

i18next scanner seems to skip multi line strings that need to be translated.

This will work as expected:

<p>{t('Lorem ipsum dolor sit amet consequetat lorem ipsum labora ecta.')}</p>

This one instead will be skipped by parser:

<p>{t('Lorem ipsum dolor sit amet consequetat ' +
'lorem ipsum labora ecta.')}</p>

Maybe the regexp used by Parser need to be changed a little bit in order to manage the \s_+\s_ pattern.

Add support to disable the keyseparator

From i18next documentation you can setup the library to support a key as a value fallback, in that case the documentation allows you to define the keyseparator option to false as follows:

http://i18next.com/translate/keyBasedFallback/

[question] about loadPath, etc.

Version

• i18next-scanner: 2.2.1

a) What doe's loadPath actually?
b) How can configure my scan, so the translations are not overwritten on every call?

I mean if I have "slide1maintext": "STRING_NOT_TRANSLATED", and I edit it. And run again to add new keys, the translations are gone :/

Double escaped quotes issue

Version

• i18next: 3.1.0
• i18next-scanner: 1.4.0

Configuration

{ 
          lngs: ['en', 'es'], // supported languages
          fallbackLng: 'en',
          ns: ['mainsite'],
          defaultNs: 'mainsite',
          defaultValue: function(lng, ns, key) {
            if (lng === 'en') {
              // Return key as the default value for English language
              return key;
            }
            // Return the string '__NOT_TRANSLATED__' for other languages
            return '__NOT_TRANSLATED__';
          },
          nsSeparator: false,
          keySeparator: false,
          resource: {
              // the source path is relative to current working directory
              loadPath: 'translations/i18n/{{lng}}/{{ns}}.json',
              // the destination path is relative to your `gulp.dest()` path
              savePath: 'i18n/{{lng}}/{{ns}}.json'
          }
}

Having this on file:

i18n.t("Primary \"email\" activation")

Produces on destination json:

{ 
  "Primary \\\"email\\\" activation": "Primary \\\"email\\\" activation"
}

i18next searches for:

{ 
  "Primary \"email\" activation": "Primary \"email\" activation"
}

JS function handling is very fragile

The parser is not very robust when it looks for translation functions. Changing test/fixtures/app.js to add some newlines breaks the tests:

i18next.t('key2');
i18next.t(
  'key1'
);

For my specific use case I would like to be able to handle this code:

i18next.t(
  'errors.password-mismatch',
  {
    defaultValue: 'Your passwords did not match.'
  }
);

Multiple Plural Forms

Dear maintainers, your libraries looks extensive, but I can't get it to work for languages which have more than one plural form. Here's my gulp task, just stolen from the example:

gulp.task('i18next', function() {
    return gulp.src(['frontend/js-webpack/react/**/*.{js,html}'])
        .pipe(scanner({
            lngs: ['en', 'de', 'ua', 'uk', 'ru'], // supported languages
            resource: {
                // the source path is relative to current working directory
                loadPath: 'assets/i18n/{{lng}}/{{ns}}.json',

                // the destination path is relative to your `gulp.dest()` path
                savePath: 'i18n/{{lng}}/{{ns}}.json'
            }
        }))
        .pipe(gulp.dest('assets'));
});

That results in the same file for all the locales, e.g. ru/translation.json:

{
  "Connected to": "",
  "Connected to_plural": ""
}

There should be keys like "Connected to_1", "Connected to_2", "Connected to_5", as described in the i18next docs. Am I missing something?

Thanks for you kind attention!

NameSpace not found

Could someone please tell why I am getting this error when I try to pass the argument "App key : "

This is the error

Please ensure the namespace "App Key" exists in the ns.namespaces[] of your i18next-scanner options: {"ns":"App Key","key":"","defaultValue":"App Key:"}

Multiple keys created when context used

Version

• i18next: 3.5.2, also occurs with 7.1.1
• i18next-scanner: 1.5.1

Configuration

{ // i18next-scanner options
	func: {
		list: ["t"],
	},
	keySeparator: false,
	nsSeparator: false,
}

This occurs with the items from the unit test "Context".

If extracting just:
i18next.t('friend', {context: 'male'});

The resulting translations get extracted:

        en: {
            translation: {
                "friend": "",
                "friend_male": "",
            }
        }

I'm unable to run the unit tests in this repo but it appears that this isn't seen because the test extracts the translations that use and don't use the context together and the issue is hidden.

Default value of t func is not extracted by Scanner.

Version

• i18next: latest
• i18next-scanner: latest

Configuration

{ // i18next-scanner options
options: {
    func: {
      list: ['i18next.t', 'i18n.t', 't'],
      extensions: ['.js', '.jsx']
    },
    lngs: ['en','de'],
    ns: [
      'login',
      'home'
    ],
    defaultLng: 'en',
    defaultNs: 'login',
    resource: {
      //loadPath: 'i18n/{{lng}}/{{ns}}.json',
      savePath: 'locales/{{lng}}/{{ns}}.json',
      jsonIndent: 2,
      lineEnding: '\n'
    },
    nsSeparator: ':',
    keySeparator: '.'
  }
}

I have posted this issue #1085 in i18next project.
Also When i have t function like this t("namespace:title",) and t(namespace:${name}), this kind of key is not extract by scanner.

I have tried i18n-parser also, both scanner and parser did not extract default value as well ass dynamic value.

This makes me to think like i have to manually maintain the resource catalog which will be tedious process.

Any help.

How can we just modify a existing file ?

Hi,
I actually use i18next-scanner with vinyl-fs as many exemple, my problem is, when i parse my main file, it overwrite with new informations (json) so I lost with that all my values.

Do you have any solution to just modify property if the file already exist ?

Cordially.

To extract `<Trans/>` strings the scanner should use the same lib to parse html as react-i18next

Under certain circumstances, the order for the indexed tags are not the same for the parsed strings within react-i18next and i18next-scanner. Translation will fallback to key, if no trans key is specified and the generated key from the string is used.

Following text:

<Trans><p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</p>
		<p>Lorem Ipsum has been the industry's standard dummy text ever since the 1500s</p>
                </Trans>

[email protected] generated key:
<0>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</0> <2>Lorem Ipsum has been the industry's standard dummy text ever since the 1500s</2>

[email protected] generated key:
<0>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</0><1>Lorem Ipsum has been the industry's standard dummy text ever since the 1500s</1>

Version

• i18next: 10.2.2
• i18next-scanner: 2.4.4

Configuration

default konfiguration supporting <trans/>.

Add webpack support

Similarly to the support for grunt and gulp
https://github.com/i18next/i18next-scanner#grunt
https://github.com/i18next/i18next-scanner#gulp
I would appreciate an easy to set up webpack plugin.

cli output path insists on i18n directory

I'm playing with the new CLI. It seems to work well, except for one thing: it always creates an i18n directory, even if you specify --output. For example when I run this command:

yarn run i18next-scanner \
    --config config/i18next-scanner.js \
    --output public/locales \
    'src/**/*.{js,jsx}'

I want the output to be stored in public/locales. Instead the output is stored in public/locales/i18n.

Support different looking Trans props

For parsing the Trans key there are multiple valid ways of doing props:

<Trans i18nKey="key" />
<Trans i18nKey='key' />
<Trans i18nKey={"key"} />
<Trans i18nKey={'key'} />

Could we add those to the regex?

Sorting of the keys when multiple levels are used

Version

• i18next: 8.4.3
• i18next-scanner: 2.0.1

Configuration

{ // i18next-scanner options
sort: true
}

When using multiple levels of the message id like: some.other.key sorting of the output json doesn't seem to work and outputs random order

Backticks in functions are not handled correctly

With current git master the following two lines of code produce some very weird results:

i18next.t(`errors.${e.errors[key].result}`)
i18next.t(`errors.${defaultError}`)

The result looks like this:

     {                                                                  
       "en": {                                                          
         "translation": {                                               
           "errors": {                                                  
             "${defaultError}": "",                                     
             "${e": {                                                   
               "errors[key]": {                                         
                 "result}": ""                                          
               }                                                        
             }                                                          
           },                                                           
         }                                                              
       }                                                                

There is a workaround: either write the code to not use backticks like so:

i18next.t('errors.' + e.errors[key].result)
i18next.t('errors.' + defaultError)

or use a temporary variable and pass that to i18next.t.

Consider parsing `ns` from function call

First, thanks for the package, its already helping us manage projects with i18next translations.

In the spirit of making translation easier on our international team and more manageable for us, I was wondering if you would consider parsing the namespace if provided?

I saw reading the source that you parse defaultValue context and count from a hypothetical option object that would be provided to a i18next.t() call, is there a reason I am missing (apart from not needing the feature so far) not to parse the namespace the same way ?

For instance :

i18next.t('This is a test string', { ns: 'notYourDefaultNS' })

Would result in the 'This is a test string' key to be added to the notYourDefaultNS namespace instead of the default one like now.

The `removeUnusedKeys` option re-orders keys in translation file

Using the removeUnusedKeys is re-ordering many items in my translation file. This makes it very difficult to diff which translations were added/removed. Disabling this option works as expected.

Version

• i18next: 10.2.1
• i18next-scanner: 2.2.1

Configuration

{
    options: {
        debug: true,
        removeUnusedKeys: true,
        sort: false,
        attr: false,
        func: {
            list: ['__'],
            extensions: ['.js', '.jsx']
        },
        ns: ['common'],
        defaultNs: 'common',
        nsSeparator: '::',
        keySeparator: false,
        contextFallback: false,
        plural: false,
        pluralFallback: false,
        interpolation: {
            prefix: '{{',
            suffix: '}}',
        },
        lngs: ['ja'],
        defaultValue: null,
        resource: {
            loadPath: 'i18n/{{lng}}/{{ns}}.json',
            savePath: 'i18n/{{lng}}/{{ns}}.json',
            jsonIndent: 4,
        },
    }
};

Trans extraction breaks when children and defaults nodes are used together

As of 2.6.1, the following JSX:

<React.Fragment>
	<Trans defaults="Some <0>{variable}</0>" />
	<Trans>Hello, World!</Trans>
</React.Fragment>

results in the following error:

{ SyntaxError: Unexpected token (1:19)
    at Parser.pp$4.raise (/opt/application/source/node_modules/acorn-jsx/node_modules/acorn/dist/acorn.js:2756:13)
    at Parser.pp.unexpected (/opt/application/source/node_modules/acorn-jsx/node_modules/acorn/dist/acorn.js:647:8)
    at Parser.pp.jsx_parseIdentifier (/opt/application/source/node_modules/acorn-jsx/inject.js:194:12)
    at Parser.pp.jsx_parseNamespacedName (/opt/application/source/node_modules/acorn-jsx/inject.js:203:21)
    at Parser.pp.jsx_parseElementName (/opt/application/source/node_modules/acorn-jsx/inject.js:218:21)
    at Parser.pp.jsx_parseClosingElementAt (/opt/application/source/node_modules/acorn-jsx/inject.js:305:25)
    at Parser.pp.jsx_parseElementAt (/opt/application/source/node_modules/acorn-jsx/inject.js:327:35)
    at Parser.pp.jsx_parseElement (/opt/application/source/node_modules/acorn-jsx/inject.js:375:17)
    at Parser.parseExprAtom (/opt/application/source/node_modules/acorn-jsx/inject.js:397:23)
    at Parser.pp$3.parseExprSubscripts (/opt/application/source/node_modules/acorn-jsx/node_modules/acorn/dist/acorn.js:2047:19) pos: 19, loc: Position { line: 1, column: 19 }, raisedAt: 20 }

and neither are extracted.

Note this only seems to happen when the defaults mode contains a tag using the <0></0> syntax.

Version

• i18next: 11.3.3
• i18next-scanner: 2.6.1

Configuration

const fs = require("fs");

module.exports = {
	options: {
		// use strings as keys
		nsSeparator: false,
		keySeparator: false,
		// settings
		defaultNs: "frontend",
		lngs: ["en"],
		resource: {
			loadPath: "{{lng}}/{{ns}}.json",
			savePath: "{{lng}}/{{ns}}.json",
		},
		func: false,
		trans: false,
		defaultValue: (language, namespace, key) => key,
	},
	transform(file, enc, done) {
		const parser = this.parser;
		const content = fs.readFileSync(file.path, enc);

		parser.parseFuncFromString(
			content,
			{
				list: ["t"],
			},
			(key, options) => {
				if (["some", "blacklisted", "keys"].indexOf(key) !== -1) {
					return;
				}
				parser.set(key, options);
			},
		);

		parser.parseTransFromString(
			content,
			{
				component: ["Trans"],
				i18nKey: false,
			},
			(key, options) => {
				parser.set(options.defaultValue, options);
			},
		);

		done();
	},
};

Expand README.md with how to merge translations

Your README.md states

Scan your code, extract translation keys/values, and merge them into i18n resource files.

But I couldn't find an example of how the "merge" part is done. How can I merge the extracted keys/values with existing translations?

I would expect it to be part of the standard-api section

Part of the values are overwritten

Version

• i18next: 10.6.0
• i18next-scanner: 2.4.6

Configuration

exports.i18nScannerOptions = {
  // debug: true,
  func: {
    list: ['t', 'props.t', 'this.props.t', 'i18next.t', 'i18n.t', 'this.props.t!'],
    extensions: ['.ts', '.tsx'],
  },
  lngs: ['en', 'zh'],
  ns: ['App','Lab']
  defaultLng: 'en',
  defaultNs: 'App',
  resource: {
    loadPath: 'translations/{{lng}}/{{ns}}.json',
    savePath: '{{lng}}/{{ns}}.json',
    jsonIndent: 2,
    lineEnding: '\n',
  },
  nsSeparator: ':', // namespace separator
  keySeparator: false, // key separator
  interpolation: {
    escapeValue: false, // not needed for react!!
    prefix: '{{',
    suffix: '}}',
  },
};

Hi there, I am running into some weird cases when using the scanner, 90% of the time it runs smoothly, but sometimes the scanner would overwrite some keys in my translation files without my notice, for example, I have 4 keys already translated

key1: "a"
key2:"b",
key3:"c",
key4:"d"

but at some unknown time, the file would turn into

key1: "a"
key2:default_value
key3:default_value,
key4:"d"

It is obvious that Some translated keys are overwritten by default value, note that I also use the parser.parseFuncFromString function to set the default value equal to the key.

Thank you!

Appending, not replacing the contents of the destination file

Version

• i18next: 11.3.2
• i18next-scanner: 2.6.5

Configuration

var fs = require("fs");
var chalk = require("chalk");

module.exports = {
  options: {
    debug: true,
    attr: false,
    removeUnusedKeys: true,
    sort: true,
    func: {
      list: ["t"], // 'i18next.t', 'i18n.t',
      extensions: [".tsx"] // '.js', '.jsx',
    },
    trans: false,
    // {
    //     extensions: ['.tsx' ],
    //     fallbackKey: (ns, value) => {
    //         return value;
    //     }
    // },
    lngs: ["en"],
    ns: ["translations", "common", "traffic"],
    defaultNs: "translations",
    defaultValue: null, // '__STRING_NOT_TRANSLATED__',
    resource: {
      loadPath: "locales/{{lng}}/{{ns}}.json",
      savePath: "locales/{{lng}}/{{ns}}.json"
    },
    nsSeparator: ":", // namespace separator
    keySeparator: ".", // key separator
    interpolation: {
      prefix: "{{",
      suffix: "}}"
    }
  },
  transform: function customTransform(file, enc, done) {
    "use strict";
    const parser = this.parser;
    console.log(`Scanning ${file.path}`);
    const content = fs.readFileSync(file.path, enc);
    let count = 0;

    parser.parseFuncFromString(
      content,
      { list: ["i18next._", "i18next.__", "t"] },
      (key, options) => {
        parser.set(
          key,
          Object.assign({}, options, {
            nsSeparator: ":",
            keySeparator: "."
          })
        );
        ++count;
      }
    );

    if (count > 0) {
      console.log(
        `i18next-scanner: count=${chalk.cyan(count)}, file=${chalk.yellow(
          JSON.stringify(file.relative)
        )}`
      );
    }

    done();
  }
};

Hi there,

Thank you for this wonderful library.

I have the configuration above. The ideal scenario for me would be:

1. The developer inserts call to i18next.t in their code with a key ns:key, i18next-scanner job is run.
2. If the job finds the key with a non null value in the file ns, then it's left alone. If not, its inserted into the file with the value null. It is the job of the developer to then provide a value for this key into the translation file.
3. If the job find any keys not used in source code they're removed from the resource file. (this seems to be done no matter what the value of removeUnusedKeys.

Essentially it seems like i18next-scanner always overrides the translation files with source code. It overwrites whatever is there, I would like it to leave alone any keys for which there are values in the translation file already, add any that are new in source code and remove any that are in translation file and not in source code.

What's the best way to achieve this with your tool?

Thank you.

-Hamid