admin/j-lang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add while loops, GC mark-and-sweep, and malloc block reuse

Browse Source 
- Lexer: add INDENT/DEDENT tokens, <, >, : operators, while keyword,
  closing DEDENT emission, include guards
- Parser: add NODE_WHILE with while_loop union, parse while/cond/body
  blocks, include guards
- Eval: add while loop evaluation, GC integration with roots from env,
  debug and gc flags, <, > comparison operators
- GC: implement mark-and-sweep collector with 3 stages (mark roots,
  sweep unmarked, join free blocks)
- Allocator: block reuse via first-fit search with splitting, exponential
  heap growth, NULL check on malloc, include guards, marked field in metadata
- Object: add include guards, fix include to use allocator.h
This commit is contained in:
Jose Luis Montañes Ojados
2026-02-16 04:55:52 +01:00
parent 14b6a2ddd2
commit 84b3abbfda
8 changed files with 542 additions and 286 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
projects/while.j Normal file
View File

@@ -0,0 +1,4 @@
x = 0
while x < 100000000:
x = x + 1
print x

38
src/frontend/lexer.h
View File

@@ -1,3 +1,5 @@
#ifndef JLANG_LEXER_H
#define JLANG_LEXER_H
/*
Convierte texto en una lista de tokens
*/
@@ -67,6 +69,8 @@ Token *tokenize(const char *source, int *token_count) {
int count = 0;
int pos = 0;
int indent_level = 0;
while (source[pos] != '\0' && pos < strlen(source)) {
char c = source[pos];
@@ -75,6 +79,20 @@ Token *tokenize(const char *source, int *token_count) {
} else if (c == '\n') {
tokens[count++] = make_token(TOK_NEWLINE, "\n");
pos++;
// Contar espacios al inicio de la linea
int spaces = 0;
while (source[pos] == ' ') {
spaces++;
pos++;
}
int new_level = spaces / 4; // 4 espacios = 1 nivel
if (new_level > indent_level) {
tokens[count++] = make_token(TOK_INDENT, "INDENT");
} else if (new_level < indent_level) {
tokens[count++] = make_token(TOK_DEDENT, "DEDENT");
}
indent_level = new_level;
} else if (c == '+') {
tokens[count++] = make_token(TOK_PLUS, "+");
pos++;
@@ -90,6 +108,15 @@ Token *tokenize(const char *source, int *token_count) {
} else if (c == '=') {
tokens[count++] = make_token(TOK_ASSIGN, "=");
pos++;
} else if (c == '<') {
tokens[count++] = make_token(TOK_LT, "<");
pos++;
} else if (c == '>') {
tokens[count++] = make_token(TOK_GT, ">");
pos++;
} else if (c == ':') {
tokens[count++] = make_token(TOK_COLON, ":");
pos++;
} else if (c >= '0' && c <= '9') {
// Leer todos los digitos consecutivos
int start = pos;
@@ -106,6 +133,8 @@ Token *tokenize(const char *source, int *token_count) {
// Comprobar si es una keyword reservada
if (strcmp(word, "print") == 0) {
tokens[count++] = make_token(TOK_PRINT, word);
} else if (strcmp(word, "while") == 0) {
tokens[count++] = make_token(TOK_WHILE, word);
} else {
tokens[count++] = make_token(TOK_ID, word);
}
@@ -115,6 +144,15 @@ Token *tokenize(const char *source, int *token_count) {
}
}
// Cerrar indents pendientes
while (indent_level > 0) {
tokens[count++] = make_token(TOK_DEDENT, "DEDENT");
indent_level--;
}
*token_count = count;
return tokens;
}
#endif

124
src/frontend/parser.h
View File

@@ -1,11 +1,12 @@
#ifndef JLANG_PARSER_H
#define JLANG_PARSER_H
/*
Convierte tokens en un arbol
*/
#include "lexer.h"
typedef enum
{
typedef enum {
NODE_INT_LIT, // literal entero
NODE_STRING_LIT, // literal string
NODE_VAR, // referencia a variable
@@ -17,38 +18,35 @@ typedef enum
NODE_BLOCK, // secuencia de statements
} NodeType;
typedef struct ASTNode
{
typedef struct ASTNode {
NodeType type;
union
{
union {
int int_val; // NODE_INT_LIT
char *string_val; // NODE_STRING_LIT
struct
{
struct {
char *name;
struct ASTNode *value;
} assign; // NODE_ASSIGN
struct
{
struct {
char op;
struct ASTNode *left;
struct ASTNode *right;
} binop; // NODE_BINOP
struct
{
struct {
struct ASTNode *expr;
} print; // NODE_PRINT
struct
{
struct {
struct ASTNode **stmts;
int count;
} block; // NODE_BLOCK
struct {
struct ASTNode *cond;
struct ASTNode *body;
} while_loop; // NODE_WHILE
} data;
} ASTNode;
ASTNode *make_node(NodeType type)
{
ASTNode *make_node(NodeType type) {
ASTNode *node = (ASTNode *)malloc(sizeof(ASTNode));
node->type = type;
return node;
@@ -56,17 +54,13 @@ ASTNode *make_node(NodeType type)
int pos = 0;
ASTNode *parse_term(Token *tokens)
{
if (tokens[pos].type == TOK_INT)
{
ASTNode *parse_term(Token *tokens) {
if (tokens[pos].type == TOK_INT) {
ASTNode *node = make_node(NODE_INT_LIT);
node->data.int_val = atoi(tokens[pos].value);
pos++;
return node;
}
else if (tokens[pos].type == TOK_ID)
{
} else if (tokens[pos].type == TOK_ID) {
ASTNode *node = make_node(NODE_VAR);
node->data.string_val = tokens[pos].value;
pos++;
@@ -76,12 +70,12 @@ ASTNode *parse_term(Token *tokens)
exit(1);
}
ASTNode *parse_expr(Token *tokens)
{
ASTNode *parse_expr(Token *tokens) {
ASTNode *left = parse_term(tokens);
while (tokens[pos].type == TOK_PLUS || tokens[pos].type == TOK_MINUS || tokens[pos].type == TOK_STAR || tokens[pos].type == TOK_SLASH)
{
while (tokens[pos].type == TOK_PLUS || tokens[pos].type == TOK_MINUS ||
tokens[pos].type == TOK_STAR || tokens[pos].type == TOK_SLASH ||
tokens[pos].type == TOK_LT || tokens[pos].type == TOK_GT) {
char op = tokens[pos].value[0]; // +,-,*,/
pos++;
ASTNode *right = parse_term(tokens);
@@ -95,10 +89,8 @@ ASTNode *parse_expr(Token *tokens)
return left;
}
ASTNode *parse_statement(Token *tokens)
{
if (tokens[pos].type == TOK_ID)
{
ASTNode *parse_statement(Token *tokens) {
if (tokens[pos].type == TOK_ID) {
char *name = tokens[pos].value;
pos++; // consumir ID
pos++; // consumir "="
@@ -113,41 +105,62 @@ ASTNode *parse_statement(Token *tokens)
pos++; // consumir "print"
ASTNode *expr = parse_expr(tokens);
ASTNode*node = make_node(NODE_PRINT);
ASTNode *node = make_node(NODE_PRINT);
node->data.print.expr = expr;
return node;
}
if (tokens[pos].type == TOK_WHILE) {
pos++; // consumir while
ASTNode *cond = parse_expr(tokens);
pos++; // consumir ":"
pos++; // consumir NEWLINE
pos++; // consumir INDENT
// Parsear bloque de statements hasta DEDENT
ASTNode *body = make_node(NODE_BLOCK);
body->data.block.stmts = (ASTNode **)malloc(sizeof(ASTNode *) * 256);
body->data.block.count = 0;
while (tokens[pos].type != TOK_DEDENT) {
body->data.block.stmts[body->data.block.count++] =
parse_statement(tokens);
if (tokens[pos].type == TOK_NEWLINE) {
pos++;
}
}
pos++; // Consumir DEDENT
ASTNode *node = make_node(NODE_WHILE);
node->data.while_loop.cond = cond;
node->data.while_loop.body = body;
return node;
}
printf("ERROR: statement inesperado\n");
exit(1);
}
ASTNode *parse(Token *tokens, int token_count)
{
ASTNode *parse(Token *tokens, int token_count) {
ASTNode *block = make_node(NODE_BLOCK);
block->data.block.stmts = (ASTNode **)malloc(sizeof(ASTNode *) * 256);
block->data.block.count = 0;
while (pos < token_count)
{
if (tokens[pos].type == TOK_NEWLINE)
{
while (pos < token_count) {
if (tokens[pos].type == TOK_NEWLINE) {
pos++; // Saltar newlines sueltos
continue;
}
block->data.block.stmts[block->data.block.count++] = parse_statement(tokens);
block->data.block.stmts[block->data.block.count++] =
parse_statement(tokens);
// Consumir newline despues del statement
if (pos < token_count && tokens[pos].type == TOK_NEWLINE)
{
if (pos < token_count && tokens[pos].type == TOK_NEWLINE) {
pos++;
}
}
return block;
}
void ast_print(ASTNode *node, const char *prefix, int is_last)
{
void ast_print(ASTNode *node, const char *prefix, int is_last) {
if (!node)
return;
@@ -156,10 +169,16 @@ void ast_print(ASTNode *node, const char *prefix, int is_last)
// Construir nuevo prefijo para hijos
char new_prefix[256];
snprintf(new_prefix, sizeof(new_prefix), "%s%s", prefix, is_last ? " " : "| ");
snprintf(new_prefix, sizeof(new_prefix), "%s%s", prefix,
is_last ? " " : "| ");
switch (node->type) {
case NODE_WHILE:
printf("NODE_WHILE\n");
ast_print(node->data.while_loop.cond, new_prefix, 0);
ast_print(node->data.while_loop.body, new_prefix, 1);
break;
switch (node->type)
{
case NODE_INT_LIT:
printf("NODE_INT_LIT(%d)\n", node->data.int_val);
break;
@@ -190,9 +209,9 @@ void ast_print(ASTNode *node, const char *prefix, int is_last)
case NODE_BLOCK:
printf("NODE_BLOCK\n");
for (int i = 0; i < node->data.block.count; i++)
{
ast_print(node->data.block.stmts[i], new_prefix, i == node->data.block.count - 1);
for (int i = 0; i < node->data.block.count; i++) {
ast_print(node->data.block.stmts[i], new_prefix,
i == node->data.block.count - 1);
}
break;
@@ -202,7 +221,6 @@ void ast_print(ASTNode *node, const char *prefix, int is_last)
}
}
void ast_debug(ASTNode *node)
{
ast_print(node, "", 1);
}
void ast_debug(ASTNode *node) { ast_print(node, "", 1); }
#endif

5
src/main.c
View File

@@ -34,7 +34,10 @@ int main(int argc, char **argv) {
ast_debug(block);
Environment env = {0};
eval(block, &env, allocPtr);
eval(block, &env, allocPtr, 0, 1);
printf("heapSize=%zu\n", allocPtr->size);
// JLANG_visualize(allocPtr);
return 0;
}

83
src/memory/allocator.h
View File

@@ -1,3 +1,6 @@
#ifndef JLANG_ALLOCATOR_H
#define JLANG_ALLOCATOR_H
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -24,7 +27,7 @@
typedef struct {
size_t size; // 8 bytes
int in_use; // 4 bytes
// --- aquí el compilador mete 4 bytes invisibles ---
int marked; // 4 bytes
} JLANG_metadata;
/*
@@ -94,6 +97,75 @@ void *JLANG_last_free(void *ptr) {
size_t JLANG_malloc(void *ptr, size_t size) {
JLANG_memory_allocator *allocPtr = (JLANG_memory_allocator *)ptr;
// Recorrer memoria hasta encontrar un hueco libre
JLANG_metadata *currentHead = (JLANG_metadata *)allocPtr->memory;
size_t used = 0;
while (1) {
used += currentHead->size;
if (currentHead->in_use == 0 && currentHead->size >= size) {
break;
}
if (currentHead->size == 0) {
size_t offset = (char *)currentHead - allocPtr->memory;
if (offset + sizeof(JLANG_metadata) + size > allocPtr->size) {
// No cabe, grow
size_t newSize = allocPtr->size * 2;
// asegurar que quepa la peticion
while (newSize < offset + sizeof(JLANG_metadata) + size) {
newSize *= 2;
}
char *newBuffer = (char *)malloc(newSize); // pedir newSize bytes
if (newBuffer == NULL) {
printf("ERROR: out of memory\n");
exit(1);
}
memset(newBuffer, 0, newSize);
memcpy(newBuffer, allocPtr->memory,
allocPtr->size); // Copiamos al nuevo buffer
allocPtr->size = newSize;
free(allocPtr->memory); // Liberamos el buffer antiguo
allocPtr->memory = newBuffer; // Seteamos el nuevo buffer
JLANG_metadata *freeHeader =
(JLANG_metadata *)(allocPtr->memory + offset);
freeHeader->size = size;
freeHeader->in_use = 1;
void *payloadPtr = (char *)freeHeader + sizeof(JLANG_metadata);
return (char *)payloadPtr - allocPtr->memory;
}
// Fin del heap, asignar aqui
currentHead->size = size;
currentHead->in_use = 1;
return (char *)currentHead - allocPtr->memory + sizeof(JLANG_metadata);
}
// Go next
currentHead =
(JLANG_metadata *)((char *)currentHead + sizeof(JLANG_metadata) +
currentHead->size);
}
// currentHead tiene sitio para nosotros, particionamos y usamos
currentHead->in_use = 1;
size_t oldSize = currentHead->size;
currentHead->size = size;
if (oldSize > size + sizeof(JLANG_metadata)) {
// creamos un bloque vacio con los bytes que sobran
JLANG_metadata *emptyBlock =
(JLANG_metadata *)((char *)currentHead + sizeof(JLANG_metadata) + size);
emptyBlock->size = oldSize - size - sizeof(JLANG_metadata);
}
return (char *)currentHead - allocPtr->memory + sizeof(JLANG_metadata);
/*
// 1. Hay bloques disponibles??
size_t used = JLANG_used(ptr);
size_t available = allocPtr->size - used;
@@ -109,11 +181,7 @@ size_t JLANG_malloc(void *ptr, size_t size) {
return (char *)payloadPtr - allocPtr->memory;
}
/*
Se necesita más espacio..
De momento vamos a pedir otro array más grande y liberar el antiguo
Mantenemos toda la memoria contigua
*/
// Se necesita más espacio..
int newSize = allocPtr->size + 1024;
char *newBuffer = (char *)malloc(newSize); // Otro 1KB
@@ -132,6 +200,7 @@ size_t JLANG_malloc(void *ptr, size_t size) {
void *payloadPtr = (char *)freeHeader + sizeof(JLANG_metadata);
return (char *)payloadPtr - allocPtr->memory;
*/
}
void JLANG_free(void *ptr, size_t blockOffset) {
@@ -254,3 +323,5 @@ void JLANG_visualize(void *ptr) {
printf("]\n");
}
}
#endif

95
src/memory/gc.h Normal file
View File

@@ -0,0 +1,95 @@
#ifndef JLANG_GC_H
#define JLANG_GC_H
#include "../objects/object.h"
#include "allocator.h"
void gc_collect(JLANG_memory_allocator *allocPtr, size_t *roots,
int root_count) {
// Stage 1. Mark blocks
for (int i = 0; i < root_count; i++) {
// get blockHeader from object offset
JLANG_metadata *blockHeader =
(JLANG_metadata *)((char *)allocPtr->memory + roots[i] -
sizeof(JLANG_metadata));
// Mark block
blockHeader->marked = 1;
Object *objPtr = (Object *)((char *)allocPtr->memory + roots[i]);
if (objPtr->type == OBJ_LIST) {
JLANG_metadata *itemsHeader =
(JLANG_metadata *)((char *)allocPtr->memory +
objPtr->data.list_val.items -
sizeof(JLANG_metadata));
itemsHeader->marked = 1;
} else if (objPtr->type == OBJ_STRING) {
JLANG_metadata *itemsHeader =
(JLANG_metadata *)((char *)allocPtr->memory +
objPtr->data.string_val.chars -
sizeof(JLANG_metadata));
itemsHeader->marked = 1;
}
}
// Stage 2. Sweep memory
JLANG_metadata *currentHead = (JLANG_metadata *)allocPtr->memory;
while (currentHead->size != 0) {
size_t blockOffset = (char *)currentHead - (char *)allocPtr->memory;
if (currentHead->marked == 0 && currentHead->in_use) {
// Free block
JLANG_free(allocPtr, blockOffset + sizeof(JLANG_metadata));
}
// Reset mark
currentHead->marked = 0;
currentHead = (JLANG_metadata *)((char *)currentHead + currentHead->size +
sizeof(JLANG_metadata));
}
// Stage 3. Join free blocks
currentHead = (JLANG_metadata *)allocPtr->memory;
while (currentHead->size != 0) {
if (currentHead->in_use == 0) {
// iterate until last next free block
JLANG_metadata *nextHeader =
(JLANG_metadata *)((char *)currentHead + sizeof(JLANG_metadata) +
currentHead->size);
JLANG_metadata *lastHeader = currentHead;
while (1) {
if (nextHeader->in_use == 1 || nextHeader->size == 0) {
// Volvemos atrás
break;
}
lastHeader = nextHeader;
nextHeader =
(JLANG_metadata *)((char *)nextHeader + sizeof(JLANG_metadata) +
nextHeader->size);
}
// Make from currentHead to nextHeader + nextHeader(size) one block
size_t startIndex = (char *)currentHead - (char *)allocPtr->memory;
size_t endIndex =
((char *)lastHeader + sizeof(JLANG_metadata) + lastHeader->size) -
(char *)allocPtr->memory;
// Set to 0
for (int i = 0; i < (endIndex - startIndex); i++) {
allocPtr->memory[i + startIndex] = 0;
}
// Create valid header
currentHead = (JLANG_metadata *) ((char *)allocPtr->memory + startIndex);
currentHead->size = (endIndex - startIndex) - sizeof(JLANG_metadata);
}
currentHead = (JLANG_metadata *)((char *)currentHead + currentHead->size +
sizeof(JLANG_metadata));
}
}
#endif

6
src/objects/object.h
View File

@@ -1,5 +1,9 @@
#ifndef JLANG_OBJECT_H
#define JLANG_OBJECT_H
#include "../memory/allocator.h"
#define JLANG_RESOLVE(alloc, offset) \
((void *)(((JLANG_memory_allocator *)(alloc))->memory + (offset)))
@@ -137,3 +141,5 @@ void obj_print(void *allocator, size_t offset, const char *preffix) {
break;
};
}
#endif

77
src/vm/eval.h
View File

@@ -1,24 +1,22 @@
#include "../objects/object.h"
#include "../frontend/parser.h"
#ifndef JLANG_EVAL_H
#define JLANG_EVAL_H
typedef struct
{
#include "../frontend/parser.h"
#include "../memory/gc.h"
typedef struct {
char *name;
size_t value; // offset al Object en el allocator
} Variable;
typedef struct
{
typedef struct {
Variable vars[256];
int count;
} Environment;
size_t env_get(Environment *env, const char *name)
{
for (int i = 0; i < env->count; i++)
{
if (strcmp(env->vars[i].name, name) == 0)
{
size_t env_get(Environment *env, const char *name) {
for (int i = 0; i < env->count; i++) {
if (strcmp(env->vars[i].name, name) == 0) {
return env->vars[i].value;
}
}
@@ -27,12 +25,9 @@ size_t env_get(Environment *env, const char *name)
exit(1);
}
void env_set(Environment *env, const char *name, size_t value)
{
for (int i = 0; i < env->count; i++)
{
if (strcmp(env->vars[i].name, name) == 0)
{
void env_set(Environment *env, const char *name, size_t value) {
for (int i = 0; i < env->count; i++) {
if (strcmp(env->vars[i].name, name) == 0) {
env->vars[i].value = value;
return;
}
@@ -46,13 +41,25 @@ void env_set(Environment *env, const char *name, size_t value)
int step = 0;
size_t eval(ASTNode *node, Environment *env, void *allocator) {
size_t eval(ASTNode *node, Environment *env, void *allocator, int debug, int gc) {
// Run GC
if (gc) {
size_t roots[256];
for (int i = 0; i < env->count; i++) {
roots[i] = env->vars[i].value;
}
gc_collect(allocator, roots, env->count);
}
if (debug > 0) {
step++;
printf("===== VM Step: %d =====\n", step);
printf("executing node:\n");
ast_debug(node);
printf("\n");
JLANG_visualize(allocator);
}
switch (node->type) {
case NODE_INT_LIT:
@@ -60,20 +67,20 @@ size_t eval(ASTNode *node, Environment *env, void *allocator) {
case NODE_VAR:
return env_get(env, node->data.string_val);
case NODE_ASSIGN: {
size_t val = eval(node->data.assign.value, env, allocator);
size_t val = eval(node->data.assign.value, env, allocator, debug, gc);
env_set(env, node->data.assign.name, val);
return val;
}
case NODE_BINOP: {
size_t left = eval(node->data.binop.left, env, allocator);
size_t right = eval(node->data.binop.right, env, allocator);
size_t left = eval(node->data.binop.left, env, allocator, debug, gc);
size_t right = eval(node->data.binop.right, env, allocator, debug, gc);
// Resolver objects
Object* l = (Object *) JLANG_RESOLVE(allocator, left);
Object* r = (Object *) JLANG_RESOLVE(allocator, right);
Object *l = (Object *)JLANG_RESOLVE(allocator, left);
Object *r = (Object *)JLANG_RESOLVE(allocator, right);
// Operar (ints por ahora)
if (node->data.binop.op == '+'){
if (node->data.binop.op == '+') {
return obj_new_int(allocator, l->data.int_val + r->data.int_val);
} else if (node->data.binop.op == '-') {
return obj_new_int(allocator, l->data.int_val - r->data.int_val);
@@ -81,19 +88,33 @@ size_t eval(ASTNode *node, Environment *env, void *allocator) {
return obj_new_int(allocator, l->data.int_val * r->data.int_val);
} else if (node->data.binop.op == '/') {
return obj_new_int(allocator, l->data.int_val / r->data.int_val);
} else if (node->data.binop.op == '<') {
return obj_new_int(allocator, l->data.int_val < r->data.int_val);
} else if (node->data.binop.op == '>') {
return obj_new_int(allocator, l->data.int_val > r->data.int_val);
}
}
case NODE_PRINT: {
size_t val = eval(node->data.print.expr, env, allocator);
size_t val = eval(node->data.print.expr, env, allocator, debug, gc);
obj_print(allocator, val, "");
printf("\n");
return val;
}
case NODE_BLOCK:
for (int i=0; i< node->data.block.count; i++)
eval(node->data.block.stmts[i], env, allocator);
for (int i = 0; i < node->data.block.count; i++)
eval(node->data.block.stmts[i], env, allocator, debug, gc);
return 0;
case NODE_WHILE:
while (1) {
size_t cond = eval(node->data.while_loop.cond, env, allocator, debug, gc);
Object *obj = (Object *)JLANG_RESOLVE(allocator, cond);
if (obj->data.int_val == 0)
break;
eval(node->data.while_loop.body, env, allocator, debug, gc);
}
default:
break;
}
}
#endif