import { StopwordsBasedTokenizer } from '../../code-search/tokenizer/StopwordsBasedTokenizer';
import { WhitespaceBasedTokenizer } from '../../code-search/tokenizer/WhitespaceBasedTokenizer';

describe('SimilarChunkTokenizer', () => {
	let tokenizer: StopwordsBasedTokenizer;

	beforeEach(() => {
		tokenizer = StopwordsBasedTokenizer.instance();
	});

	describe('instance', () => {
		it('should return an instance of SimilarChunkTokenizer', () => {
			expect(tokenizer).to.be.instanceOf(StopwordsBasedTokenizer);
		});

		it('should always return the same instance', () => {
			const anotherInstance = StopwordsBasedTokenizer.instance();
			expect(anotherInstance).to.equal(tokenizer);
		});
	});

	describe('tokenize', () => {
		it('should return a set of unique words from the input string, excluding stop words and programming keywords', () => {
			const input = 'this is a test function for SimilarChunkTokenizer';
			const expectedOutput = new Set(['chunk', 'similar', 'tokenizer', 'test', 'similarchunktokenizer']);
			expect(tokenizer.tokenize(input)).to.deep.equal(expectedOutput);
		});

		// java hello world
		it('should split the input string into an array of words', () => {
			const input = `public class HelloWorld {
      public static void main(String[] args) {
          System.out.println("Hello, World");
      }
  }`;
			const expectedOutput: Set<string> = new Set([
				'helloworld',
				'void',
				'main',
				'string',
				'args',
				'system',
				'println',
				'hello',
				'world',
			]);
			expect(tokenizer.tokenize(input)).to.deep.equal(expectedOutput);
		});
	});

	describe('splitIntoWords', () => {
		it('should split the input string into an array of words', () => {
			const input = 'this is a test';
			const expectedOutput: Set<string> = new Set(['this', 'is', 'a', 'test']);
			let tokenizer = new WhitespaceBasedTokenizer();
			expect(tokenizer.tokenize(input)).to.deep.equal(expectedOutput);
		});
	});
});