Lines Matching refs:tokens
194 static int16_t tokens[LEADBYTE_LIMIT*256];
225 tokens[i]=0;
508 if(tokens[i]==-1) {
516 /* do we need double-byte tokens? */
518 /* no, single-byte tokens are enough */
521 if(tokens[i]!=-1) {
522 tokens[i]=wordNumber;
524 printf("tokens[0x%03x]: word%8ld \"%.*s\"\n",
534 * The tokens that need two token bytes
553 /* how many tokens and lead bytes do we have now? */
557 * double-byte tokens must not
571 /* adjust again to skip double-byte tokens with ';' */
576 tokens[0]=0;
578 printf("tokens[0x000]: word%8ld \"%.*s\"\n",
584 /* set the lead byte tokens */
586 tokens[i]=-2;
589 /* set the tokens */
591 /* if store10Names then the parser set tokens[NAME_SEPARATOR_CHAR]=-1 */
592 if(tokens[i]!=-1) {
593 tokens[i]=wordNumber;
595 printf("tokens[0x%03x]: word%8ld \"%.*s\"\n",
606 tokens[i]=-1; /* do not use NAME_SEPARATOR_CHAR as a second token byte */
608 tokens[i]=wordNumber;
610 printf("tokens[0x%03x]: word%8ld \"%.*s\"\n",
622 printf("number of single-byte tokens: %lu\n",
624 printf("number of tokens: %lu\n", (unsigned long)tokenCount);
675 /* write characters and tokens for this line */
786 token=tokens[i];
788 tokens[i]=(int16_t)(addToken(words[token].s, words[token].length)-groupTop);
807 * This should be the only -1 entry in tokens[256..511] on which the data
818 printf("number of tokens[] padding entries for data swapping: %lu\n", (unsigned long)(tokenCount-i));
822 tokens[i]=-1; /* do not use NAME_SEPARATOR_CHAR as a second token byte */
824 tokens[i]=0; /* unused token for padding */
835 * - the number of tokens, uint16_t (2)
870 udata_writeBlock(pData, tokens, 2*tokenCount);
922 token=tokens[i];
1033 tokens[stringStart[--length]]=-1;