Merge pull request #8664 from ntrrgc/lunr-tokenizer

Add lunr.tokenizer methods and properties

lunr/lunr.d.ts

@@ -15,11 +15,64 @@ declare namespace lunr
 
 
     /**
-     * A function for splitting a string into tokens ready to be inserted into the search index.
+     * A function for splitting a string into tokens ready to be inserted into
+     * the search index. Uses `lunr.tokenizer.seperator` to split strings, change
+     * the value of this property to change how strings are split into tokens.
      *
-     * @param token  The token to pass through the filter
+     * @module
+     * @param {String} obj The string to convert into tokens
+     * @see lunr.tokenizer.seperator
+     * @returns {Array}
      */
-    function tokenizer(token:string):string;
+    function tokenizer(obj: any): string[];
+
+    interface TokenizerFunction {
+        // obj is usually a string, but the default lunr tokenizer handles null,
+        // undefined and arrays of objects with a .toString() method.
+        (obj: any): string[];
+    }
+
+    module tokenizer {
+        /**
+         * The sperator used to split a string into tokens. Override this property to change the behaviour of
+         * `lunr.tokenizer` behaviour when tokenizing strings. By default this splits on whitespace and hyphens.
+         *
+         * @static
+         * @see lunr.tokenizer
+         *
+         * (Note: this is misspelled in the original API, kept for compatibility sake)
+         */
+        var seperator: RegExp | string;
+
+        var label: string;
+
+        var registeredFunctions: {[label: string]: TokenizerFunction};
+
+        /**
+         * Register a tokenizer function.
+         *
+         * Functions that are used as tokenizers should be registered if they are to be used with a serialised index.
+         *
+         * Registering a function does not add it to an index, functions must still be associated with a specific index for them to be used when indexing and searching documents.
+         *
+         * @param {Function} fn The function to register.
+         * @param {String} label The label to register this function with
+         * @memberOf tokenizer
+         */
+        function registerFunction(fn: TokenizerFunction, label: string): void;
+
+        /**
+         * Loads a previously serialised tokenizer.
+         *
+         * A tokenizer function to be loaded must already be registered with lunr.tokenizer.
+         * If the serialised tokenizer has not been registered then an error will be thrown.
+         *
+         * @param {String} label The label of the serialised tokenizer.
+         * @returns {Function}
+         * @memberOf tokenizer
+         */
+        function load(label: string): TokenizerFunction;
+    }
 
 
     /**