Lines Matching refs:token
46 * token=tokenTable[c];
47 * if(token==0xfffe) -- lead byte of double-byte token
48 * token=tokenTable[c<<8|next character];
49 * if(token==-1)
52 * tokenString=tokenStrings+token; (tokenStrings=start of names data + tokenStringOffset;)
76 * For the de-tokenizing, see token description above; the strings immediately follow the
179 int32_t weight; /* -(cost for token) + (number of occurences) * (length-1) */
505 /* count the letters in the token range */
534 * The tokens that need two token bytes
575 /* set token 0 to word 0 */
606 tokens[i]=-1; /* do not use NAME_SEPARATOR_CHAR as a second token byte */
703 int16_t start, limit, token, groupTop=*pGroupTop;
717 /* write a word, as token or directly */
722 token=findToken(s+start, (int16_t)(limit-start));
723 if(token!=-1) {
724 if(token>0xff) {
725 groupStore[groupTop++]=(uint8_t)(token>>8);
727 groupStore[groupTop++]=(uint8_t)token;
782 int16_t token;
784 /* first, see how much space we need, and prepare the token strings */
786 token=tokens[i];
787 if(token!=-1 && token!=-2) {
788 tokens[i]=(int16_t)(addToken(words[token].s, words[token].length)-groupTop);
794 * The token table undergoes a permutation during data swapping when the
796 * The token table cannot grow during swapping, so we need to make sure that
822 tokens[i]=-1; /* do not use NAME_SEPARATOR_CHAR as a second token byte */
824 tokens[i]=0; /* unused token for padding */
830 * - the offset to the token strings, uint32_t (4)
836 * - the token table, uint16_t[tokenCount] (2*tokenCount)
838 * - the token strings, each zero-terminated (tokenSize=(lineTop-groupTop)), 2-padded
856 "total data length %lu, token strings %lu, compressed strings %lu, algorithmic names %lu\n",
868 /* token table */
872 /* token strings */
919 int16_t i, token;
922 token=tokens[i];
923 if(token>=0 && length==words[token].length && 0==uprv_memcmp(s, words[token].s, length)) {
960 * Initialize the weight with the costs for this token:
975 /* add to the weight the savings: the length of the word minus 1 byte for the token */
1031 /* prevent a character value that is actually in a name from becoming a token */