Add comment to note use of previous token index

gunthercox · gunthercox · commit bcabc43228a4 · 2025-10-19T17:47:30.000-04:00
For #1793
diff --git a/chatterbot/components.py b/chatterbot/components.py
@@ -29,6 +29,9 @@ def chatterbot_bigram_indexer(document):
             token for token in document if not (token.is_punct)
         ]
 
+    # Pairs consist of the part-of-speech of the first token and the
+    # lemma of the second token in the bigram. This provides a good
+    # balance of generalization and specificity for matching.
     bigram_pairs = [
         f"{tokens[i - 1].pos_}:{tokens[i].lemma_.lower()}"
         for i in range(1, len(tokens))

Original file line number	Diff line number	Diff line change
`@@ -29,6 +29,9 @@ def chatterbot_bigram_indexer(document):`
`29`	`29`	`token for token in document if not (token.is_punct)`
`30`	`30`	`]`
`31`	`31`
	`32`	`+ # Pairs consist of the part-of-speech of the first token and the`
	`33`	`+ # lemma of the second token in the bigram. This provides a good`
	`34`	`+ # balance of generalization and specificity for matching.`
`32`	`35`	`bigram_pairs = [`
`33`	`36`	`f"{tokens[i - 1].pos_}:{tokens[i].lemma_.lower()}"`
`34`	`37`	`for i in range(1, len(tokens))`