token_table.h

/*
** Thread-safe Token Table
**
** Provides concurrent token lookup via reader-writer locking.
** Multiple readers can look up tokens simultaneously; writers
** acquire exclusive access for modifications.  A version counter
** is maintained for external change detection.
*/
#ifndef TOKEN_TABLE_H
#define TOKEN_TABLE_H
 
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include <stdatomic.h>
#include <pthread.h>
 
#define INVALID_INDEX 0xFFFFFFFF
 
typedef uint32_t ExtensionID;
 
typedef struct TokenDefinition {
    const char *lexeme;          
    size_t lexeme_len;           
    int token_code;              
    ExtensionID extension_id;    
    uint32_t next_in_chain;      
} TokenDefinition;
 
typedef struct TokenTable {
    TokenDefinition *tokens;     
    uint32_t ntokens;            
    uint32_t capacity;           
    uint32_t *hash_table;        
    uint32_t hash_capacity;      
    atomic_uint_fast32_t version;
    pthread_rwlock_t lock;       
} TokenTable;
 
/*
** Create a new token table with the given initial capacity.
** Returns NULL on allocation failure.
*/
TokenTable *create_token_table(uint32_t initial_capacity);
 
/*
** Destroy a token table and free all associated memory.
*/
void destroy_token_table(TokenTable *table);
 
/*
** Look up a token by string. Acquires read lock internally, so
** multiple threads can look up tokens concurrently.
** Returns the token_code if found, or -1 if not found.
*/
int lookup_token(TokenTable *table, const char *str, size_t len);
 
/*
** Add a token to the table. Acquires write lock internally.
** Returns true on success, false on failure (allocation error or duplicate).
*/
bool add_token(TokenTable *table, const char *lexeme, int token_code,
               ExtensionID ext_id);
 
/*
** Remove all tokens belonging to a given extension.
** Acquires write lock internally. Rebuilds hash chains.
** Returns true on success, false on failure.
*/
bool remove_tokens_by_extension(TokenTable *table, ExtensionID ext_id);
 
#endif /* TOKEN_TABLE_H */