mirror of
https://github.com/kuhyx/testsAndMisc-archive.git
synced 2026-07-04 13:23:01 +02:00
603 lines
17 KiB
C
603 lines
17 KiB
C
/*
|
|
* Vocabulary Learning Curve Analyzer
|
|
*
|
|
* For each excerpt length (1, 2, 3, ... N words), finds the excerpt that
|
|
* requires the minimum number of top-frequency words to understand 100%.
|
|
*
|
|
* Usage:
|
|
* ./vocabulary_curve <file.txt> [max_length]
|
|
* ./vocabulary_curve test.txt 50
|
|
*/
|
|
|
|
#include <ctype.h>
|
|
#include <stdbool.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#define MAX_WORD_LEN 64
|
|
#define MAX_WORDS 500000
|
|
#define MAX_UNIQUE_WORDS 100000
|
|
#define HASH_SIZE 200003 /* Prime number for better distribution */
|
|
|
|
/* Word entry for hash table */
|
|
typedef struct WordEntry
|
|
{
|
|
char word[MAX_WORD_LEN];
|
|
int count;
|
|
int rank; /* 1-indexed rank by frequency (1 = most common) */
|
|
struct WordEntry *next;
|
|
} WordEntry;
|
|
|
|
/* Hash table for word lookup */
|
|
static WordEntry *hash_table[HASH_SIZE];
|
|
static WordEntry *all_entries[MAX_UNIQUE_WORDS];
|
|
static int num_unique_words = 0;
|
|
|
|
/* All words in order of appearance - store POINTERS not indices */
|
|
static WordEntry *word_sequence[MAX_WORDS];
|
|
static int num_words = 0;
|
|
|
|
/* Result for each excerpt length */
|
|
typedef struct
|
|
{
|
|
int excerpt_length;
|
|
int min_vocab_needed;
|
|
int start_pos; /* Start position in word_sequence */
|
|
} ExcerptResult;
|
|
|
|
/* Simple hash function */
|
|
static unsigned int hash_word(const char *word)
|
|
{
|
|
unsigned int hash = 5381;
|
|
int c;
|
|
while ((c = *word++))
|
|
{
|
|
hash = ((hash << 5) + hash) + c;
|
|
}
|
|
return hash % HASH_SIZE;
|
|
}
|
|
|
|
/* Find or create word entry */
|
|
static WordEntry *get_or_create_word(const char *word)
|
|
{
|
|
unsigned int h = hash_word(word);
|
|
WordEntry *entry = hash_table[h];
|
|
|
|
while (entry)
|
|
{
|
|
if (strcmp(entry->word, word) == 0)
|
|
{
|
|
return entry;
|
|
}
|
|
entry = entry->next;
|
|
}
|
|
|
|
/* Create new entry */
|
|
if (num_unique_words >= MAX_UNIQUE_WORDS)
|
|
{
|
|
fprintf(stderr, "Too many unique words\n");
|
|
exit(1);
|
|
}
|
|
|
|
entry = malloc(sizeof(WordEntry));
|
|
if (!entry)
|
|
{
|
|
fprintf(stderr, "Memory allocation failed\n");
|
|
exit(1);
|
|
}
|
|
|
|
strncpy(entry->word, word, MAX_WORD_LEN - 1);
|
|
entry->word[MAX_WORD_LEN - 1] = '\0';
|
|
entry->count = 0;
|
|
entry->rank = 0;
|
|
entry->next = hash_table[h];
|
|
hash_table[h] = entry;
|
|
|
|
all_entries[num_unique_words++] = entry;
|
|
|
|
return entry;
|
|
}
|
|
|
|
/* Compare function for sorting by frequency (descending) */
|
|
static int compare_by_count(const void *a, const void *b)
|
|
{
|
|
const WordEntry *wa = *(const WordEntry **)a;
|
|
const WordEntry *wb = *(const WordEntry **)b;
|
|
return wb->count - wa->count; /* Descending */
|
|
}
|
|
|
|
/* Check if character is part of a word */
|
|
static bool is_word_char(int c) { return isalnum(c) || c == '_' || (unsigned char)c >= 128; }
|
|
|
|
/* Read and process file */
|
|
static bool process_file(const char *filename)
|
|
{
|
|
FILE *fp = fopen(filename, "r");
|
|
if (!fp)
|
|
{
|
|
fprintf(stderr, "Cannot open file: %s\n", filename);
|
|
return false;
|
|
}
|
|
|
|
char word[MAX_WORD_LEN];
|
|
int word_len = 0;
|
|
int c;
|
|
|
|
while ((c = fgetc(fp)) != EOF)
|
|
{
|
|
if (is_word_char(c))
|
|
{
|
|
if (word_len < MAX_WORD_LEN - 1)
|
|
{
|
|
word[word_len++] = tolower(c);
|
|
}
|
|
}
|
|
else if (word_len > 0)
|
|
{
|
|
word[word_len] = '\0';
|
|
|
|
WordEntry *entry = get_or_create_word(word);
|
|
entry->count++;
|
|
|
|
if (num_words >= MAX_WORDS)
|
|
{
|
|
fprintf(stderr, "Too many words in file\n");
|
|
fclose(fp);
|
|
return false;
|
|
}
|
|
|
|
/* Store pointer directly - survives sorting */
|
|
word_sequence[num_words++] = entry;
|
|
|
|
word_len = 0;
|
|
}
|
|
}
|
|
|
|
/* Handle last word if file doesn't end with whitespace */
|
|
if (word_len > 0)
|
|
{
|
|
word[word_len] = '\0';
|
|
WordEntry *entry = get_or_create_word(word);
|
|
entry->count++;
|
|
|
|
if (num_words < MAX_WORDS)
|
|
{
|
|
word_sequence[num_words++] = entry;
|
|
}
|
|
}
|
|
|
|
fclose(fp);
|
|
return true;
|
|
}
|
|
|
|
/* Assign ranks based on frequency */
|
|
static void assign_ranks(void)
|
|
{
|
|
/* Sort all_entries by frequency (this doesn't affect word_sequence) */
|
|
qsort(all_entries, num_unique_words, sizeof(WordEntry *), compare_by_count);
|
|
|
|
/* Assign 1-indexed ranks using competition ranking:
|
|
* Words with same frequency get same rank.
|
|
* Next rank is current_position + 1 (skipping numbers).
|
|
* Example: counts 5,3,3,2 -> ranks 1,2,2,4 (not 1,2,3,4) */
|
|
for (int i = 0; i < num_unique_words; i++)
|
|
{
|
|
if (i == 0)
|
|
{
|
|
all_entries[i]->rank = 1;
|
|
}
|
|
else if (all_entries[i]->count == all_entries[i - 1]->count)
|
|
{
|
|
/* Same frequency as previous word - same rank */
|
|
all_entries[i]->rank = all_entries[i - 1]->rank;
|
|
}
|
|
else
|
|
{
|
|
/* Different frequency - rank is position + 1 */
|
|
all_entries[i]->rank = i + 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Analyze excerpt and return max rank needed */
|
|
static int analyze_excerpt(int start, int length)
|
|
{
|
|
/* Track which entries we've seen using a simple visited array */
|
|
/* We use the rank field is already assigned, so we can check uniqueness */
|
|
static bool seen_rank[MAX_UNIQUE_WORDS + 1];
|
|
memset(seen_rank, 0, (num_unique_words + 1) * sizeof(bool));
|
|
|
|
int max_rank = 0;
|
|
|
|
for (int i = start; i < start + length; i++)
|
|
{
|
|
WordEntry *entry = word_sequence[i];
|
|
int rank = entry->rank;
|
|
|
|
if (!seen_rank[rank])
|
|
{
|
|
seen_rank[rank] = true;
|
|
if (rank > max_rank)
|
|
{
|
|
max_rank = rank;
|
|
}
|
|
}
|
|
}
|
|
|
|
return max_rank;
|
|
}
|
|
|
|
/* Find optimal excerpts for each length */
|
|
static void find_optimal_excerpts(int max_length, ExcerptResult *results)
|
|
{
|
|
for (int length = 1; length <= max_length && length <= num_words; length++)
|
|
{
|
|
int best_vocab = num_unique_words + 1;
|
|
int best_start = 0;
|
|
|
|
/* Slide window through text */
|
|
for (int start = 0; start <= num_words - length; start++)
|
|
{
|
|
int vocab_needed = analyze_excerpt(start, length);
|
|
|
|
if (vocab_needed < best_vocab)
|
|
{
|
|
best_vocab = vocab_needed;
|
|
best_start = start;
|
|
}
|
|
}
|
|
|
|
results[length - 1].excerpt_length = length;
|
|
results[length - 1].min_vocab_needed = best_vocab;
|
|
results[length - 1].start_pos = best_start;
|
|
}
|
|
}
|
|
|
|
/* Print excerpt words */
|
|
static void print_excerpt(int start, int length)
|
|
{
|
|
for (int i = start; i < start + length; i++)
|
|
{
|
|
if (i > start)
|
|
printf(" ");
|
|
printf("%s", word_sequence[i]->word);
|
|
}
|
|
}
|
|
|
|
/* Print words needed (sorted by rank) */
|
|
static void print_words_needed(int start, int length)
|
|
{
|
|
/* Collect unique entries */
|
|
static WordEntry *unique_entries[MAX_UNIQUE_WORDS];
|
|
static bool seen_rank[MAX_UNIQUE_WORDS + 1];
|
|
memset(seen_rank, 0, (num_unique_words + 1) * sizeof(bool));
|
|
|
|
int count = 0;
|
|
for (int i = start; i < start + length; i++)
|
|
{
|
|
WordEntry *entry = word_sequence[i];
|
|
if (!seen_rank[entry->rank])
|
|
{
|
|
seen_rank[entry->rank] = true;
|
|
unique_entries[count++] = entry;
|
|
}
|
|
}
|
|
|
|
/* Sort by rank (simple bubble sort - small arrays) */
|
|
for (int i = 0; i < count - 1; i++)
|
|
{
|
|
for (int j = i + 1; j < count; j++)
|
|
{
|
|
if (unique_entries[i]->rank > unique_entries[j]->rank)
|
|
{
|
|
WordEntry *tmp = unique_entries[i];
|
|
unique_entries[i] = unique_entries[j];
|
|
unique_entries[j] = tmp;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Print */
|
|
for (int i = 0; i < count; i++)
|
|
{
|
|
if (i > 0)
|
|
printf(", ");
|
|
printf("%s(#%d)", unique_entries[i]->word, unique_entries[i]->rank);
|
|
}
|
|
}
|
|
|
|
/* Print results */
|
|
static void print_results(ExcerptResult *results, int max_length)
|
|
{
|
|
printf("======================================================================\n");
|
|
printf("VOCABULARY LEARNING CURVE\n");
|
|
printf("======================================================================\n");
|
|
printf("\n");
|
|
printf("For each excerpt length, the minimum number of top-frequency\n");
|
|
printf("words you need to learn to understand 100%% of some excerpt.\n");
|
|
printf("\n");
|
|
printf("Total words in text: %d\n", num_words);
|
|
printf("Unique words: %d\n", num_unique_words);
|
|
printf("\n");
|
|
printf("----------------------------------------------------------------------\n");
|
|
|
|
int prev_vocab = 0;
|
|
int actual_max = max_length;
|
|
if (actual_max > num_words)
|
|
actual_max = num_words;
|
|
|
|
for (int i = 0; i < actual_max; i++)
|
|
{
|
|
ExcerptResult *r = &results[i];
|
|
|
|
printf("\n[Length %d] Vocab needed: %d", r->excerpt_length, r->min_vocab_needed);
|
|
if (r->min_vocab_needed > prev_vocab)
|
|
{
|
|
printf(" (+%d)", r->min_vocab_needed - prev_vocab);
|
|
}
|
|
printf("\n");
|
|
|
|
printf(" Excerpt: \"");
|
|
print_excerpt(r->start_pos, r->excerpt_length);
|
|
printf("\"\n");
|
|
|
|
printf(" Words: ");
|
|
print_words_needed(r->start_pos, r->excerpt_length);
|
|
printf("\n");
|
|
|
|
prev_vocab = r->min_vocab_needed;
|
|
}
|
|
|
|
printf("\n----------------------------------------------------------------------\n");
|
|
|
|
if (actual_max > 0)
|
|
{
|
|
ExcerptResult *final = &results[actual_max - 1];
|
|
printf("\nTo understand a %d-word excerpt,\n", final->excerpt_length);
|
|
printf("you need to learn at minimum %d top words.\n", final->min_vocab_needed);
|
|
}
|
|
}
|
|
|
|
/* Free memory */
|
|
static void cleanup(void)
|
|
{
|
|
for (int i = 0; i < num_unique_words; i++)
|
|
{
|
|
free(all_entries[i]);
|
|
}
|
|
}
|
|
|
|
/* Dump all vocabulary with ranks (for Python integration) */
|
|
static void dump_vocabulary(int max_rank)
|
|
{
|
|
printf("VOCAB_DUMP_START\n");
|
|
for (int i = 0; i < num_unique_words; i++)
|
|
{
|
|
if (all_entries[i]->rank <= max_rank)
|
|
{
|
|
printf("%s;%d\n", all_entries[i]->word, all_entries[i]->rank);
|
|
}
|
|
}
|
|
printf("VOCAB_DUMP_END\n");
|
|
}
|
|
|
|
/* Find longest excerpt using only top N words (inverse mode) */
|
|
static void find_longest_excerpt(int max_vocab)
|
|
{
|
|
/* Sliding window: find longest contiguous sequence where all words have rank <= max_vocab */
|
|
int best_start = 0;
|
|
int best_length = 0;
|
|
|
|
int left = 0;
|
|
for (int right = 0; right < num_words; right++)
|
|
{
|
|
/* If current word is outside our vocabulary, move left past it */
|
|
if (word_sequence[right]->rank > max_vocab)
|
|
{
|
|
left = right + 1;
|
|
}
|
|
else
|
|
{
|
|
/* Current window [left, right] is valid */
|
|
int length = right - left + 1;
|
|
if (length > best_length)
|
|
{
|
|
best_length = length;
|
|
best_start = left;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Print results */
|
|
printf("======================================================================\n");
|
|
printf("INVERSE MODE: LONGEST EXCERPT WITH TOP %d WORDS\n", max_vocab);
|
|
printf("======================================================================\n");
|
|
printf("\n");
|
|
printf("Total words in text: %d\n", num_words);
|
|
printf("Unique words: %d\n", num_unique_words);
|
|
printf("Vocabulary limit: top %d words\n", max_vocab);
|
|
printf("\n");
|
|
printf("----------------------------------------------------------------------\n");
|
|
printf("\n");
|
|
|
|
if (best_length == 0)
|
|
{
|
|
printf("No valid excerpt found with top %d words.\n", max_vocab);
|
|
printf("The text may require rarer words from the very beginning.\n");
|
|
}
|
|
else
|
|
{
|
|
printf("LONGEST EXCERPT: %d words\n", best_length);
|
|
printf("Position: words %d to %d\n", best_start + 1, best_start + best_length);
|
|
printf("\n");
|
|
printf("Excerpt:\n \"");
|
|
print_excerpt(best_start, best_length);
|
|
printf("\"\n");
|
|
printf("\n");
|
|
|
|
/* Find the rarest word in the excerpt */
|
|
int max_rank_used = 0;
|
|
const char *rarest_word = NULL;
|
|
for (int i = best_start; i < best_start + best_length; i++)
|
|
{
|
|
if (word_sequence[i]->rank > max_rank_used)
|
|
{
|
|
max_rank_used = word_sequence[i]->rank;
|
|
rarest_word = word_sequence[i]->word;
|
|
}
|
|
}
|
|
printf("Rarest word used: %s (#%d)\n", rarest_word, max_rank_used);
|
|
|
|
/* Count unique words in excerpt */
|
|
static bool seen_rank[MAX_UNIQUE_WORDS + 1];
|
|
memset(seen_rank, 0, (num_unique_words + 1) * sizeof(bool));
|
|
int unique_count = 0;
|
|
for (int i = best_start; i < best_start + best_length; i++)
|
|
{
|
|
if (!seen_rank[word_sequence[i]->rank])
|
|
{
|
|
seen_rank[word_sequence[i]->rank] = true;
|
|
unique_count++;
|
|
}
|
|
}
|
|
printf("Unique words in excerpt: %d\n", unique_count);
|
|
}
|
|
|
|
printf("\n----------------------------------------------------------------------\n");
|
|
}
|
|
|
|
int main(int argc, char *argv[])
|
|
{
|
|
if (argc < 2)
|
|
{
|
|
fprintf(stderr, "Usage: %s <file.txt> [options]\n", argv[0]);
|
|
fprintf(stderr, "\nModes:\n");
|
|
fprintf(stderr, " (default) Find minimum vocab needed for each excerpt length\n");
|
|
fprintf(stderr,
|
|
" --max-vocab N INVERSE: Find longest excerpt using only top N words\n");
|
|
fprintf(stderr, "\nOptions:\n");
|
|
fprintf(stderr, " max_length Maximum excerpt length to analyze (default: 30)\n");
|
|
fprintf(stderr, " --dump-vocab [N] Output all words with ranks up to N\n");
|
|
fprintf(stderr, "\nExamples:\n");
|
|
fprintf(stderr, " %s book.txt 50 # Analyze excerpts up to 50 words\n",
|
|
argv[0]);
|
|
fprintf(stderr, " %s book.txt --max-vocab 500 # Find longest excerpt with top 500 words\n",
|
|
argv[0]);
|
|
return 1;
|
|
}
|
|
|
|
const char *filename = argv[1];
|
|
int max_length = 30;
|
|
bool dump_vocab = false;
|
|
int dump_max_rank = 0;
|
|
int max_vocab_mode = 0; /* 0 = normal mode, >0 = inverse mode with this vocab limit */
|
|
|
|
/* Parse arguments */
|
|
for (int i = 2; i < argc; i++)
|
|
{
|
|
if (strcmp(argv[i], "--dump-vocab") == 0)
|
|
{
|
|
dump_vocab = true;
|
|
if (i + 1 < argc && argv[i + 1][0] != '-')
|
|
{
|
|
dump_max_rank = atoi(argv[++i]);
|
|
}
|
|
}
|
|
else if (strcmp(argv[i], "--max-vocab") == 0)
|
|
{
|
|
if (i + 1 < argc)
|
|
{
|
|
max_vocab_mode = atoi(argv[++i]);
|
|
if (max_vocab_mode < 1)
|
|
{
|
|
fprintf(stderr, "Error: --max-vocab requires a positive number\n");
|
|
return 1;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
fprintf(stderr, "Error: --max-vocab requires a number\n");
|
|
return 1;
|
|
}
|
|
}
|
|
else if (argv[i][0] != '-')
|
|
{
|
|
max_length = atoi(argv[i]);
|
|
if (max_length < 1)
|
|
max_length = 1;
|
|
if (max_length > 1000)
|
|
max_length = 1000;
|
|
}
|
|
}
|
|
|
|
/* Initialize hash table */
|
|
memset(hash_table, 0, sizeof(hash_table));
|
|
|
|
/* Process file */
|
|
if (!process_file(filename))
|
|
{
|
|
return 1;
|
|
}
|
|
|
|
if (num_words == 0)
|
|
{
|
|
fprintf(stderr, "No words found in file\n");
|
|
return 1;
|
|
}
|
|
|
|
/* Assign ranks by frequency */
|
|
assign_ranks();
|
|
|
|
/* Inverse mode: find longest excerpt with limited vocabulary */
|
|
if (max_vocab_mode > 0)
|
|
{
|
|
find_longest_excerpt(max_vocab_mode);
|
|
|
|
/* Dump vocabulary if requested */
|
|
if (dump_vocab)
|
|
{
|
|
if (dump_max_rank == 0)
|
|
dump_max_rank = max_vocab_mode;
|
|
dump_vocabulary(dump_max_rank);
|
|
}
|
|
|
|
cleanup();
|
|
return 0;
|
|
}
|
|
|
|
/* Normal mode: find optimal excerpts */
|
|
ExcerptResult *results = malloc(max_length * sizeof(ExcerptResult));
|
|
if (!results)
|
|
{
|
|
fprintf(stderr, "Memory allocation failed\n");
|
|
cleanup();
|
|
return 1;
|
|
}
|
|
|
|
find_optimal_excerpts(max_length, results);
|
|
|
|
/* Print results */
|
|
print_results(results, max_length);
|
|
|
|
/* Dump vocabulary if requested */
|
|
if (dump_vocab)
|
|
{
|
|
/* If no max_rank specified, use the max from the excerpt */
|
|
if (dump_max_rank == 0 && max_length > 0)
|
|
{
|
|
dump_max_rank = results[max_length - 1].min_vocab_needed;
|
|
}
|
|
if (dump_max_rank > 0)
|
|
{
|
|
dump_vocabulary(dump_max_rank);
|
|
}
|
|
}
|
|
|
|
/* Cleanup */
|
|
free(results);
|
|
cleanup();
|
|
|
|
return 0;
|
|
}
|