harec

[hare] Hare compiler, written in C11 for POSIX OSs
Log | Files | Refs | README | LICENSE

gen.c (112243B)


      1 #include <assert.h>
      2 #include <stdlib.h>
      3 #include <string.h>
      4 #include "check.h"
      5 #include "expr.h"
      6 #include "gen.h"
      7 #include "scope.h"
      8 #include "typedef.h"
      9 #include "types.h"
     10 #include "util.h"
     11 
     12 #define EXPR_GEN_VALUE -1
     13 
     14 static const struct gen_value gv_void = {
     15 	.kind = GV_CONST,
     16 	.type = &builtin_type_void,
     17 };
     18 
     19 static struct gen_value gen_expr(struct gen_context *ctx,
     20 	const struct expression *expr);
     21 static void gen_expr_at(struct gen_context *ctx,
     22 	const struct expression *expr,
     23 	struct gen_value out);
     24 static struct gen_value gen_expr_with(struct gen_context *ctx,
     25 	const struct expression *expr,
     26 	struct gen_value *out);
     27 static void gen_global_decl(struct gen_context *ctx,
     28 	const struct declaration *decl);
     29 
     30 static void
     31 gen_defers(struct gen_context *ctx, struct gen_scope *scope)
     32 {
     33 	if (ctx->deferring || !scope) {
     34 		return;
     35 	}
     36 	ctx->deferring = true;
     37 	if (scope->defers) {
     38 		pushc(ctx->current, "gen defers");
     39 	}
     40 	for (struct gen_defer *defer = scope->defers; defer;
     41 			defer = defer->next) {
     42 		gen_expr(ctx, defer->expr);
     43 	}
     44 	ctx->deferring = false;
     45 }
     46 
     47 static struct gen_scope *
     48 gen_scope_lookup(struct gen_context *ctx, const struct scope *which)
     49 {
     50 	for (struct gen_scope *scope = ctx->scope;
     51 			scope; scope = scope->parent) {
     52 		if (scope->scope == which) {
     53 			return scope;
     54 		}
     55 	}
     56 	abort();
     57 }
     58 
     59 static struct gen_scope *
     60 push_scope(struct gen_context *ctx, const struct scope *scope)
     61 {
     62 	struct gen_scope *new = xcalloc(1, sizeof(struct gen_scope));
     63 	new->parent = ctx->scope;
     64 	new->scope = scope;
     65 	ctx->scope = new;
     66 	return new;
     67 }
     68 
     69 static void
     70 pop_scope(struct gen_context *ctx, bool gendefers)
     71 {
     72 	if (gendefers) {
     73 		gen_defers(ctx, ctx->scope);
     74 	}
     75 	struct gen_scope *scope = ctx->scope;
     76 	ctx->scope = scope->parent;
     77 	for (struct gen_defer *defer = scope->defers; defer; /* n/a */) {
     78 		struct gen_defer *next = defer->next;
     79 		free(defer);
     80 		defer = next;
     81 	}
     82 	free(scope);
     83 }
     84 
     85 static void
     86 gen_copy_memcpy(struct gen_context *ctx,
     87 	struct gen_value dest, struct gen_value src)
     88 {
     89 	struct qbe_value rtfunc = mkrtfunc(ctx, "rt.memcpy");
     90 	struct qbe_value dtemp = mklval(ctx, &dest);
     91 	struct qbe_value stemp = mklval(ctx, &src);
     92 	struct qbe_value sz = constl(dest.type->size);
     93 	pushi(ctx->current, NULL, Q_CALL, &rtfunc, &dtemp, &stemp, &sz, NULL);
     94 }
     95 
     96 static void
     97 gen_copy_aligned(struct gen_context *ctx,
     98 	struct gen_value dest, struct gen_value src)
     99 {
    100 	if (dest.type->size > 128) {
    101 		gen_copy_memcpy(ctx, dest, src);
    102 		return;
    103 	}
    104 	enum qbe_instr load, store;
    105 	assert(dest.type->align && (dest.type->align & (dest.type->align - 1)) == 0);
    106 	switch (dest.type->align) {
    107 	case 1: load = Q_LOADUB, store = Q_STOREB; break;
    108 	case 2: load = Q_LOADUH, store = Q_STOREH; break;
    109 	case 4: load = Q_LOADUW, store = Q_STOREW; break;
    110 	default:
    111 		assert(dest.type->align == 8);
    112 		load = Q_LOADL, store = Q_STOREL;
    113 		break;
    114 	}
    115 	struct qbe_value temp = {
    116 		.kind = QV_TEMPORARY,
    117 		.type = ctx->arch.ptr,
    118 		.name = gen_name(ctx, ".%d"),
    119 	};
    120 	struct qbe_value destp = mkcopy(ctx, &dest, ".%d");
    121 	struct qbe_value srcp = mkcopy(ctx, &src, ".%d");
    122 	struct qbe_value align = constl(dest.type->align);
    123 	for (size_t offset = 0; offset < dest.type->size;
    124 			offset += dest.type->align) {
    125 		pushi(ctx->current, &temp, load, &srcp, NULL);
    126 		pushi(ctx->current, NULL, store, &temp, &destp, NULL);
    127 		if (offset + dest.type->align < dest.type->size) {
    128 			pushi(ctx->current, &srcp, Q_ADD, &srcp, &align, NULL);
    129 			pushi(ctx->current, &destp, Q_ADD, &destp, &align, NULL);
    130 		}
    131 	}
    132 }
    133 
    134 static void
    135 gen_store(struct gen_context *ctx,
    136 	struct gen_value object,
    137 	struct gen_value value)
    138 {
    139 	const struct type *ty = type_dealias(object.type);
    140 	switch (ty->storage) {
    141 	case STORAGE_ARRAY:
    142 	case STORAGE_SLICE:
    143 	case STORAGE_STRING:
    144 	case STORAGE_STRUCT:
    145 	case STORAGE_TAGGED:
    146 	case STORAGE_TUPLE:
    147 	case STORAGE_VALIST:
    148 		gen_copy_aligned(ctx, object, value);
    149 		return;
    150 	case STORAGE_UNION:
    151 		gen_copy_memcpy(ctx, object, value);
    152 		return;
    153 	case STORAGE_ENUM:
    154 		object.type = ty->alias.type;
    155 		break;
    156 	default:
    157 		break; // no-op
    158 	}
    159 
    160 	struct qbe_value qobj = mkqval(ctx, &object),
    161 		qval = mkqval(ctx, &value);
    162 	enum qbe_instr qi = store_for_type(ctx, object.type);
    163 	pushi(ctx->current, NULL, qi, &qval, &qobj, NULL);
    164 }
    165 
    166 static struct gen_value
    167 gen_load(struct gen_context *ctx, struct gen_value object)
    168 {
    169 	const struct type *ty = type_dealias(object.type);
    170 	switch (ty->storage) {
    171 	case STORAGE_ARRAY:
    172 	case STORAGE_FUNCTION:
    173 	case STORAGE_SLICE:
    174 	case STORAGE_STRING:
    175 	case STORAGE_STRUCT:
    176 	case STORAGE_TAGGED:
    177 	case STORAGE_TUPLE:
    178 	case STORAGE_UNION:
    179 	case STORAGE_VALIST:
    180 		return object;
    181 	case STORAGE_ENUM:
    182 		object.type = ty->alias.type;
    183 		break;
    184 	default:
    185 		break; // no-op
    186 	}
    187 
    188 	struct gen_value value = {
    189 		.kind = GV_TEMP,
    190 		.type = object.type,
    191 		.name = gen_name(ctx, "load.%d"),
    192 	};
    193 	struct qbe_value qobj = mkqval(ctx, &object),
    194 		qval = mkqval(ctx, &value);
    195 	enum qbe_instr qi = load_for_type(ctx, object.type);
    196 	pushi(ctx->current, &qval, qi, &qobj, NULL);
    197 	return value;
    198 }
    199 
    200 static void
    201 gen_fixed_abort(struct gen_context *ctx,
    202 	struct location loc, enum fixed_aborts reason)
    203 {
    204 	int n = snprintf(NULL, 0, "%s:%d:%d",
    205 			sources[loc.file], loc.lineno, loc.colno);
    206 	char *s = xcalloc(1, n + 1);
    207 	snprintf(s, n, "%s:%d:%d",
    208 			sources[loc.file], loc.lineno, loc.colno);
    209 	struct expression eloc = {0};
    210 	eloc.type = EXPR_CONSTANT;
    211 	eloc.result = &builtin_type_const_str;
    212 	eloc.constant.string.value = s;
    213 	eloc.constant.string.len = n - 1;
    214 	struct gen_value msg = gen_expr(ctx, &eloc);
    215 	struct qbe_value qmsg = mkqval(ctx, &msg);
    216 	struct qbe_value rtabort = mkrtfunc(ctx, "rt.abort_fixed");
    217 	struct qbe_value tmp = constl(reason);
    218 	pushi(ctx->current, NULL, Q_CALL, &rtabort, &qmsg, &tmp, NULL);
    219 }
    220 
    221 static struct gen_value
    222 gen_autoderef(struct gen_context *ctx, struct gen_value val)
    223 {
    224 	while (type_dealias(val.type)->storage == STORAGE_POINTER) {
    225 		val.type = type_dealias(val.type)->pointer.referent;
    226 		val = gen_load(ctx, val);
    227 	}
    228 	return val;
    229 }
    230 
    231 static struct gen_value
    232 gen_access_ident(struct gen_context *ctx, const struct expression *expr)
    233 {
    234 	const struct scope_object *obj = expr->access.object;
    235 	switch (obj->otype) {
    236 	case O_BIND:
    237 		for (const struct gen_binding *gb = ctx->bindings;
    238 				gb; gb = gb->next) {
    239 			if (gb->object == obj) {
    240 				return gb->value;
    241 			}
    242 		}
    243 		break;
    244 	case O_DECL:
    245 		return (struct gen_value){
    246 			.kind = GV_GLOBAL,
    247 			.type = obj->type,
    248 			.name = ident_to_sym(&obj->ident),
    249 		};
    250 	case O_CONST:
    251 	case O_TYPE:
    252 	case O_SCAN:
    253 		abort(); // Invariant
    254 	}
    255 	abort(); // Invariant
    256 }
    257 
    258 static struct gen_value
    259 gen_access_index(struct gen_context *ctx, const struct expression *expr)
    260 {
    261 	struct gen_value glval = gen_expr(ctx, expr->access.array);
    262 	glval = gen_autoderef(ctx, glval);
    263 	struct qbe_value qlval = mkqval(ctx, &glval);
    264 	struct qbe_value qival = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    265 	bool checkbounds = true;
    266 	struct qbe_value length;
    267 	const struct type *ty = type_dealias(glval.type);
    268 	switch (ty->storage) {
    269 	case STORAGE_SLICE:;
    270 		enum qbe_instr load = load_for_type(ctx, &builtin_type_size);
    271 		struct qbe_value base = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    272 		pushi(ctx->current, &base, load, &qlval, NULL);
    273 
    274 		struct qbe_value temp = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    275 		length = mkqtmp(ctx, ctx->arch.sz, "len.%d");
    276 		struct qbe_value offset = constl(builtin_type_size.size);
    277 		pushi(ctx->current, &temp, Q_ADD, &qlval, &offset, NULL);
    278 		pushi(ctx->current, &length, load, &temp, NULL);
    279 
    280 		qlval = base;
    281 		break;
    282 	case STORAGE_ARRAY:
    283 		if (ty->array.length != SIZE_UNDEFINED) {
    284 			length = constl(ty->array.length);
    285 		} else {
    286 			checkbounds = false;
    287 		}
    288 		break;
    289 	default:
    290 		assert(0); // Unreachable
    291 	}
    292 
    293 	struct gen_value index = gen_expr(ctx, expr->access.index);
    294 	struct qbe_value qindex = mkqval(ctx, &index);
    295 	struct qbe_value itemsz = constl(expr->result->size);
    296 	pushi(ctx->current, &qival, Q_MUL, &qindex, &itemsz, NULL);
    297 	pushi(ctx->current, &qival, Q_ADD, &qlval, &qival, NULL);
    298 
    299 	if (checkbounds) {
    300 		struct qbe_value valid = mkqtmp(ctx, &qbe_word, ".%d");
    301 		pushi(ctx->current, &valid, Q_CULTL, &qindex, &length, NULL);
    302 
    303 		struct qbe_statement linvalid, lvalid;
    304 		struct qbe_value binvalid = mklabel(ctx, &linvalid, ".%d");
    305 		struct qbe_value bvalid = mklabel(ctx, &lvalid, ".%d");
    306 
    307 		pushi(ctx->current, NULL, Q_JNZ, &valid, &bvalid, &binvalid, NULL);
    308 		push(&ctx->current->body, &linvalid);
    309 		gen_fixed_abort(ctx, expr->loc, ABORT_OOB);
    310 		push(&ctx->current->body, &lvalid);
    311 	}
    312 
    313 	return (struct gen_value){
    314 		.kind = GV_TEMP,
    315 		.type = expr->result,
    316 		.name = qival.name,
    317 	};
    318 }
    319 
    320 static struct gen_value
    321 gen_access_field(struct gen_context *ctx, const struct expression *expr)
    322 {
    323 	const struct struct_field *field = expr->access.field;
    324 	struct gen_value glval = gen_expr(ctx, expr->access._struct);
    325 	glval = gen_autoderef(ctx, glval);
    326 	struct qbe_value qlval = mkqval(ctx, &glval);
    327 	struct qbe_value qfval = mkqtmp(ctx, ctx->arch.ptr, "field.%d");
    328 	struct qbe_value offs = constl(field->offset);
    329 	pushi(ctx->current, &qfval, Q_ADD, &qlval, &offs, NULL);
    330 	return (struct gen_value){
    331 		.kind = GV_TEMP,
    332 		.type = field->type,
    333 		.name = qfval.name,
    334 	};
    335 }
    336 
    337 static struct gen_value
    338 gen_access_value(struct gen_context *ctx, const struct expression *expr)
    339 {
    340 	const struct type_tuple *tuple = expr->access.tvalue;
    341 	struct gen_value glval = gen_expr(ctx, expr->access.tuple);
    342 	glval = gen_autoderef(ctx, glval);
    343 	struct qbe_value qlval = mkqval(ctx, &glval);
    344 	struct qbe_value qfval = mkqtmp(ctx, ctx->arch.ptr, "value.%d");
    345 	struct qbe_value offs = constl(tuple->offset);
    346 	pushi(ctx->current, &qfval, Q_ADD, &qlval, &offs, NULL);
    347 	return (struct gen_value){
    348 		.kind = GV_TEMP,
    349 		.type = tuple->type,
    350 		.name = qfval.name,
    351 	};
    352 }
    353 
    354 static struct gen_value
    355 gen_expr_access_addr(struct gen_context *ctx, const struct expression *expr)
    356 {
    357 	struct gen_value addr;
    358 	switch (expr->access.type) {
    359 	case ACCESS_IDENTIFIER:
    360 		addr = gen_access_ident(ctx, expr);
    361 		break;
    362 	case ACCESS_INDEX:
    363 		addr = gen_access_index(ctx, expr);
    364 		break;
    365 	case ACCESS_FIELD:
    366 		addr = gen_access_field(ctx, expr);
    367 		break;
    368 	case ACCESS_TUPLE:
    369 		addr = gen_access_value(ctx, expr);
    370 		break;
    371 	}
    372 	return addr;
    373 }
    374 
    375 static struct gen_value
    376 gen_expr_access(struct gen_context *ctx, const struct expression *expr)
    377 {
    378 	struct gen_value addr = gen_expr_access_addr(ctx, expr);
    379 	return gen_load(ctx, addr);
    380 }
    381 
    382 static void
    383 gen_alloc_slice_at(struct gen_context *ctx,
    384 		const struct expression *expr,
    385 		struct gen_value out,
    386 		bool expand)
    387 {
    388 	struct qbe_value qcap;
    389 	if (expr->alloc.cap) {
    390 		struct gen_value cap = gen_expr(ctx, expr->alloc.cap);
    391 		qcap = mkqval(ctx, &cap);
    392 	}
    393 
    394 	struct gen_value init;
    395 	struct qbe_value qinit;
    396 	struct qbe_value length, initdata;
    397 	const struct type *inittype = type_dealias(expr->alloc.init->result);
    398 	switch (inittype->storage) {
    399 	case STORAGE_ARRAY:
    400 		assert(inittype->array.length != SIZE_UNDEFINED);
    401 		length = constl(inittype->array.length);
    402 		break;
    403 	case STORAGE_SLICE:
    404 		init = gen_expr(ctx, expr->alloc.init);
    405 		qinit = mkqval(ctx, &init);
    406 		enum qbe_instr load = load_for_type(ctx, &builtin_type_size);
    407 		initdata = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    408 		pushi(ctx->current, &initdata, load, &qinit, NULL);
    409 		struct qbe_value offset = constl(builtin_type_size.size);
    410 		struct qbe_value ptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    411 		pushi(ctx->current, &ptr, Q_ADD, &qinit, &offset, NULL);
    412 		length = mkqtmp(ctx, ctx->arch.sz, ".%d");
    413 		pushi(ctx->current, &length, load, &ptr, NULL);
    414 		break;
    415 	default: abort(); // Invariant
    416 	}
    417 
    418 	if (!expr->alloc.cap) {
    419 		qcap = length;
    420 	}
    421 
    422 	const struct type *sltype = type_dealias(expr->result);
    423 	struct qbe_value isize = constl(sltype->array.members->size);
    424 	struct qbe_value size = mkqtmp(ctx, ctx->arch.sz, ".%d");
    425 	pushi(ctx->current, &size, Q_MUL, &qcap, &isize, NULL);
    426 
    427 	struct qbe_statement lzero, lnonzero;
    428 	struct qbe_value bzero = mklabel(ctx, &lzero, ".%d");
    429 	struct qbe_value bnonzero = mklabel(ctx, &lnonzero, ".%d");
    430 
    431 	struct qbe_value rtfunc = mkrtfunc(ctx, "rt.malloc");
    432 	struct qbe_value data = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    433 	struct qbe_value cmpres = mkqtmp(ctx, &qbe_word, ".%d");
    434 	struct qbe_value zero = constl(0);
    435 	pushi(ctx->current, &data, Q_COPY, &zero, NULL);
    436 	pushi(ctx->current, &cmpres, Q_CNEL, &size, &zero, NULL);
    437 	pushi(ctx->current, NULL, Q_JNZ, &cmpres, &bnonzero, &bzero, NULL);
    438 	push(&ctx->current->body, &lnonzero);
    439 	pushi(ctx->current, &data, Q_CALL, &rtfunc, &size, NULL);
    440 
    441 	struct qbe_statement linvalid;
    442 	struct qbe_value binvalid = mklabel(ctx, &linvalid, ".%d");
    443 	pushi(ctx->current, NULL, Q_JNZ, &data, &bzero, &binvalid, NULL);
    444 	push(&ctx->current->body, &linvalid);
    445 	gen_fixed_abort(ctx, expr->loc, ABORT_ALLOC_FAILURE);
    446 	push(&ctx->current->body, &lzero);
    447 
    448 	struct qbe_value base = mklval(ctx, &out);
    449 	struct qbe_value ptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    450 	struct qbe_value offset = constl(builtin_type_size.size);
    451 	enum qbe_instr store = store_for_type(ctx, &builtin_type_size);
    452 	pushi(ctx->current, NULL, store, &data, &base, NULL);
    453 	pushi(ctx->current, &ptr, Q_ADD, &base, &offset, NULL);
    454 
    455 	if (expand) {
    456 		pushi(ctx->current, NULL, store, &qcap, &ptr, NULL);
    457 	} else {
    458 		pushi(ctx->current, NULL, store, &length, &ptr, NULL);
    459 	}
    460 
    461 	offset = constl(builtin_type_size.size * 2);
    462 	pushi(ctx->current, &ptr, Q_ADD, &base, &offset, NULL);
    463 	pushi(ctx->current, NULL, store, &qcap, &ptr, NULL);
    464 
    465 	if (inittype->storage == STORAGE_ARRAY) {
    466 		struct gen_value storage = (struct gen_value){
    467 			.kind = GV_TEMP,
    468 			.type = inittype,
    469 			.name = data.name,
    470 		};
    471 		gen_expr_at(ctx, expr->alloc.init, storage);
    472 	} else {
    473 		struct qbe_value rtmemcpy = mkrtfunc(ctx, "rt.memcpy");
    474 		pushi(ctx->current, NULL, Q_CALL, &rtmemcpy,
    475 				&data, &initdata, &size, NULL);
    476 	}
    477 
    478 	if (!expand) {
    479 		return;
    480 	}
    481 
    482 	struct qbe_value last = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    483 	struct qbe_value next = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    484 	pushi(ctx->current, &next, Q_MUL, &length, &isize, NULL);
    485 	pushi(ctx->current, &next, Q_ADD, &next, &data, NULL);
    486 	pushi(ctx->current, &last, Q_SUB, &next, &isize, NULL);
    487 	struct qbe_value remain = mkqtmp(ctx, ctx->arch.sz, ".%d");
    488 	pushi(ctx->current, &remain, Q_SUB, &qcap, &length, NULL);
    489 	pushi(ctx->current, &remain, Q_MUL, &remain, &isize, NULL);
    490 	struct qbe_value rtmemcpy = mkrtfunc(ctx, "rt.memcpy");
    491 	pushi(ctx->current, NULL, Q_CALL, &rtmemcpy, &next, &last, &remain, NULL);
    492 }
    493 
    494 static struct gen_value
    495 gen_expr_alloc_init_with(struct gen_context *ctx,
    496 	const struct expression *expr, struct gen_value *out)
    497 {
    498 	// alloc(init) case
    499 	assert(expr->alloc.cap == NULL);
    500 
    501 	const struct type *objtype = type_dealias(expr->result);
    502 	assert(objtype->storage == STORAGE_POINTER);
    503 	objtype = objtype->pointer.referent;
    504 
    505 	struct qbe_value sz = constl(objtype->size);
    506 	struct gen_value result = mkgtemp(ctx, expr->result, ".%d");
    507 	struct qbe_value qresult = mkqval(ctx, &result);
    508 	struct qbe_value rtfunc = mkrtfunc(ctx, "rt.malloc");
    509 	pushi(ctx->current, &qresult, Q_CALL, &rtfunc, &sz, NULL);
    510 
    511 	if (!(type_dealias(expr->result)->pointer.flags & PTR_NULLABLE)) {
    512 		struct qbe_statement linvalid, lvalid;
    513 		struct qbe_value binvalid = mklabel(ctx, &linvalid, ".%d");
    514 		struct qbe_value bvalid = mklabel(ctx, &lvalid, ".%d");
    515 
    516 		pushi(ctx->current, NULL, Q_JNZ, &qresult, &bvalid, &binvalid, NULL);
    517 		push(&ctx->current->body, &linvalid);
    518 		gen_fixed_abort(ctx, expr->loc, ABORT_ALLOC_FAILURE);
    519 		push(&ctx->current->body, &lvalid);
    520 	}
    521 
    522 	struct gen_value object = {
    523 		.kind = GV_TEMP,
    524 		.type = objtype,
    525 		.name = result.name,
    526 	};
    527 	gen_expr_at(ctx, expr->alloc.init, object);
    528 	if (out) {
    529 		gen_store(ctx, *out, result);
    530 	}
    531 	return result;
    532 }
    533 
    534 static struct gen_value
    535 gen_expr_alloc_slice_with(struct gen_context *ctx,
    536 	const struct expression *expr, struct gen_value *out)
    537 {
    538 	// alloc(init, cap | len) case
    539 	bool expand = expr->alloc.kind == ALLOC_WITH_LEN;
    540 	if (out) {
    541 		gen_alloc_slice_at(ctx, expr, *out, expand);
    542 		return gv_void;
    543 	}
    544 	struct gen_value temp = mkgtemp(ctx, expr->result, "object.%d");
    545 	struct qbe_value base = mkqval(ctx, &temp);
    546 	struct qbe_value sz = constl(expr->result->size);
    547 	enum qbe_instr alloc = alloc_for_align(expr->result->align);
    548 	pushprei(ctx->current, &base, alloc, &sz, NULL);
    549 	gen_alloc_slice_at(ctx, expr, temp, expand);
    550 	return temp;
    551 }
    552 
    553 static struct gen_value
    554 gen_expr_alloc_copy_with(struct gen_context *ctx,
    555 	const struct expression *expr, struct gen_value *out)
    556 {
    557 	// alloc(init...) case
    558 	assert(expr->alloc.cap == NULL);
    559 
    560 	struct gen_value ret = gv_void;
    561 	if (out == NULL) {
    562 		ret = mkgtemp(ctx, expr->result, "object.%d");
    563 		out = &ret;
    564 
    565 		struct qbe_value base = mkqval(ctx, out);
    566 		struct qbe_value sz = constl(expr->result->size);
    567 		enum qbe_instr alloc = alloc_for_align(expr->result->align);
    568 		pushprei(ctx->current, &base, alloc, &sz, NULL);
    569 	}
    570 
    571 	enum qbe_instr load = load_for_type(ctx, &builtin_type_size);
    572 	enum qbe_instr store = store_for_type(ctx, &builtin_type_size);
    573 
    574 	struct gen_value src = gen_expr(ctx, expr->alloc.init);
    575 	struct qbe_value dbase = mkcopy(ctx, out, ".%d");
    576 	struct qbe_value offs = constl(builtin_type_size.size);
    577 
    578 	const struct type *initres = type_dealias(expr->alloc.init->result);
    579 	struct qbe_value srcdata;
    580 	struct qbe_value length;
    581 	if (initres->storage == STORAGE_SLICE) {
    582 		assert(initres->array.length == SIZE_UNDEFINED);
    583 		srcdata = mkqtmp(ctx, ctx->arch.sz, ".%d");
    584 		struct qbe_value sbase = mkcopy(ctx, &src, ".%d");
    585 		pushi(ctx->current, &srcdata, load, &sbase, NULL);
    586 		pushi(ctx->current, &sbase, Q_ADD, &sbase, &offs, NULL);
    587 		length = mkqtmp(ctx, ctx->arch.sz, ".%d");
    588 		pushi(ctx->current, &length, load, &sbase, NULL);
    589 	} else if (initres->storage == STORAGE_ARRAY) {
    590 		assert(initres->array.length != SIZE_UNDEFINED);
    591 		srcdata = mkcopy(ctx, &src, ".%d"); // TODO: object.%d
    592 		length = constl(initres->array.length);
    593 	} else {
    594 		abort();
    595 	}
    596 
    597 	struct qbe_statement linvalid, lvalid, lalloc;
    598 	struct qbe_value balloc = mklabel(ctx, &lalloc, ".%d");
    599 	struct qbe_value binvalid = mklabel(ctx, &linvalid, ".%d");
    600 	struct qbe_value bvalid = mklabel(ctx, &lvalid, ".%d");
    601 
    602 	struct qbe_value newdata = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    603 	struct qbe_value zero = constl(0);
    604 	pushi(ctx->current, &newdata, Q_COPY, &zero, NULL);
    605 	pushi(ctx->current, NULL, Q_JNZ, &length, &balloc, &bvalid, NULL);
    606 	push(&ctx->current->body, &lalloc);
    607 
    608 	const struct type *result = type_dealias(expr->result);
    609 	assert(result->storage == STORAGE_SLICE);
    610 	struct qbe_value sz = mkqtmp(ctx, ctx->arch.sz, ".%d");
    611 	struct qbe_value membsz = constl(result->array.members->size);
    612 	pushi(ctx->current, &sz, Q_MUL, &membsz, &length, NULL);
    613 
    614 	struct qbe_value rtfunc = mkrtfunc(ctx, "rt.malloc");
    615 	pushi(ctx->current, &newdata, Q_CALL, &rtfunc, &sz, NULL);
    616 
    617 	pushi(ctx->current, NULL, Q_JNZ, &newdata, &bvalid, &binvalid, NULL);
    618 	push(&ctx->current->body, &linvalid);
    619 	gen_fixed_abort(ctx, expr->loc, ABORT_ALLOC_FAILURE);
    620 	push(&ctx->current->body, &lvalid);
    621 
    622 	pushi(ctx->current, NULL, store, &newdata, &dbase, NULL);
    623 	pushi(ctx->current, &dbase, Q_ADD, &dbase, &offs, NULL);
    624 	pushi(ctx->current, NULL, store, &length, &dbase, NULL);
    625 	pushi(ctx->current, &dbase, Q_ADD, &dbase, &offs, NULL);
    626 	pushi(ctx->current, NULL, store, &length, &dbase, NULL);
    627 
    628 	struct qbe_value rtmemcpy = mkrtfunc(ctx, "rt.memcpy");
    629 	pushi(ctx->current, NULL, Q_CALL, &rtmemcpy, &newdata, &srcdata, &sz, NULL);
    630 	return ret;
    631 }
    632 
    633 static struct gen_value
    634 gen_expr_alloc_with(struct gen_context *ctx,
    635 	const struct expression *expr, struct gen_value *out)
    636 {
    637 	switch (expr->alloc.kind) {
    638 	case ALLOC_OBJECT:
    639 		return gen_expr_alloc_init_with(ctx, expr, out);
    640 	case ALLOC_WITH_CAP:
    641 	case ALLOC_WITH_LEN:
    642 		return gen_expr_alloc_slice_with(ctx, expr, out);
    643 	case ALLOC_COPY:
    644 		return gen_expr_alloc_copy_with(ctx, expr, out);
    645 	}
    646 	abort(); // Unreachable
    647 }
    648 
    649 static struct gen_value
    650 gen_expr_append(struct gen_context *ctx, const struct expression *expr)
    651 {
    652 	struct gen_value slice = gen_expr(ctx, expr->append.object);
    653 	slice = gen_autoderef(ctx, slice);
    654 	struct qbe_value qslice = mkqval(ctx, &slice);
    655 
    656 	struct qbe_value lenptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    657 	struct qbe_value prevlen = mkqtmp(ctx, ctx->arch.sz, ".%d");
    658 	struct qbe_value offs = constl(builtin_type_size.size);
    659 	pushi(ctx->current, &lenptr, Q_ADD, &qslice, &offs, NULL);
    660 	enum qbe_instr load = load_for_type(ctx, &builtin_type_size);
    661 	pushi(ctx->current, &prevlen, load, &lenptr, NULL);
    662 
    663 	struct qbe_value appendlen;
    664 	const struct type *valtype = type_dealias(expr->append.value->result);
    665 	if (expr->append.length != NULL) {
    666 		struct gen_value length = gen_expr(ctx, expr->append.length);
    667 		appendlen = mkqval(ctx, &length);
    668 		assert(valtype->storage == STORAGE_ARRAY && valtype->array.expandable);
    669 	} else if (!expr->append.is_multi) {
    670 		appendlen = constl(1);
    671 	}
    672 
    673 	struct gen_value value;
    674 	struct qbe_value qvalue;
    675 	if (!expr->append.is_multi || valtype->storage != STORAGE_ARRAY) {
    676 		// We use gen_expr_at for the array case to avoid a copy
    677 		value = gen_expr(ctx, expr->append.value);
    678 		qvalue = mkqval(ctx, &value);
    679 	}
    680 
    681 	if (expr->append.is_multi) {
    682 		if (valtype->storage == STORAGE_ARRAY) {
    683 			assert(valtype->array.length != SIZE_UNDEFINED);
    684 			appendlen = constl(valtype->array.length);
    685 		} else {
    686 			value = gen_expr(ctx, expr->append.value);
    687 			qvalue = mkqval(ctx, &value);
    688 			appendlen = mkqtmp(ctx, ctx->arch.sz, ".%d");
    689 			struct qbe_value ptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    690 			offs = constl(builtin_type_size.size);
    691 			pushi(ctx->current, &ptr, Q_ADD, &qvalue, &offs, NULL);
    692 			pushi(ctx->current, &appendlen, load, &ptr, NULL);
    693 		}
    694 	}
    695 
    696 	struct qbe_value newlen = mkqtmp(ctx, ctx->arch.sz, ".%d");
    697 	pushi(ctx->current, &newlen, Q_ADD, &prevlen, &appendlen, NULL);
    698 	enum qbe_instr store = store_for_type(ctx, &builtin_type_size);
    699 	pushi(ctx->current, NULL, store, &newlen, &lenptr, NULL);
    700 
    701 	struct qbe_value ptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    702 	const struct type *mtype = type_dealias(slice.type)->array.members;
    703 	struct qbe_value membsz = constl(mtype->size);
    704 	if (!expr->append.is_static) {
    705 		struct qbe_value rtfunc = mkrtfunc(ctx, "rt.ensure");
    706 		struct qbe_value lval = mklval(ctx, &slice);
    707 		pushi(ctx->current, NULL, Q_CALL, &rtfunc, &lval, &membsz, NULL);
    708 	} else {
    709 		offs = constl(builtin_type_size.size * 2);
    710 		pushi(ctx->current, &ptr, Q_ADD, &qslice, &offs, NULL);
    711 		struct qbe_value cap = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    712 		pushi(ctx->current, &cap, load, &ptr, NULL);
    713 
    714 		struct qbe_statement lvalid, linvalid;
    715 		struct qbe_value bvalid = mklabel(ctx, &lvalid, ".%d");
    716 		struct qbe_value binvalid = mklabel(ctx, &linvalid, ".%d");
    717 		struct qbe_value valid = mkqtmp(ctx, &qbe_word, ".%d");
    718 		pushi(ctx->current, &valid, Q_CULEL, &newlen, &cap, NULL);
    719 		pushi(ctx->current, NULL, Q_JNZ, &valid, &bvalid, &binvalid, NULL);
    720 
    721 		push(&ctx->current->body, &linvalid);
    722 		gen_fixed_abort(ctx, expr->loc, ABORT_OOB);
    723 		push(&ctx->current->body, &lvalid);
    724 	}
    725 
    726 	offs = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    727 	pushi(ctx->current, &ptr, load, &qslice, NULL);
    728 	pushi(ctx->current, &offs, Q_MUL, &prevlen, &membsz, NULL);
    729 	pushi(ctx->current, &ptr, Q_ADD, &ptr, &offs, NULL);
    730 
    731 	struct gen_value item = {
    732 		.kind = GV_TEMP,
    733 		.type = mtype,
    734 		.name = ptr.name,
    735 	};
    736 	if (expr->append.is_multi && valtype->storage == STORAGE_ARRAY) {
    737 		item.type = valtype;
    738 		gen_expr_at(ctx, expr->append.value, item);
    739 	} else if (expr->append.is_multi && valtype->storage == STORAGE_SLICE) {
    740 		struct qbe_value qsrc = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    741 		struct qbe_value rtfunc = mkrtfunc(ctx, "rt.memmove");
    742 		struct qbe_value sz = mkqtmp(ctx, ctx->arch.sz, ".%d");
    743 		pushi(ctx->current, &sz, Q_MUL, &appendlen, &membsz, NULL);
    744 		pushi(ctx->current, &qsrc, load, &qvalue, NULL);
    745 		pushi(ctx->current, NULL, Q_CALL, &rtfunc, &ptr, &qsrc, &sz, NULL);
    746 	} else if (expr->append.length != NULL) {
    747 		// XXX: This could be made more efficient for some cases if
    748 		// check could determine the length at compile time and lower it
    749 		// to a fixed-length array type
    750 		assert(valtype->storage == STORAGE_ARRAY);
    751 		item.type = valtype;
    752 		gen_expr_at(ctx, expr->append.value, item);
    753 
    754 		assert(valtype->array.length != SIZE_UNDEFINED);
    755 		struct qbe_value next = mkqtmp(ctx, ctx->arch.ptr, "next.%d");
    756 		struct qbe_value last = mkqtmp(ctx, ctx->arch.ptr, "last.%d");
    757 		struct qbe_value arlen = constl(valtype->array.length * mtype->size);
    758 		pushi(ctx->current, &next, Q_ADD, &ptr, &arlen, NULL);
    759 		arlen = constl((valtype->array.length - 1) * mtype->size);
    760 		pushi(ctx->current, &last, Q_ADD, &ptr, &arlen, NULL);
    761 
    762 		struct qbe_value rtfunc = mkrtfunc(ctx, "rt.memcpy");
    763 		struct qbe_value remain = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    764 		struct qbe_value one = constl(1);
    765 		pushi(ctx->current, &remain, Q_SUB, &appendlen, &one, NULL);
    766 		pushi(ctx->current, &remain, Q_MUL, &remain, &membsz, NULL);
    767 		pushi(ctx->current, NULL, Q_CALL, &rtfunc, &next, &last, &remain, NULL);
    768 	} else {
    769 		gen_store(ctx, item, value);
    770 	}
    771 
    772 	return gv_void;
    773 }
    774 
    775 static struct gen_value
    776 gen_expr_assert(struct gen_context *ctx, const struct expression *expr)
    777 {
    778 	assert(expr->assert.message); // Invariant
    779 	if (expr->assert.is_static) {
    780 		return gv_void;
    781 	}
    782 
    783 	struct gen_value msg;
    784 	struct qbe_statement failedl, passedl;
    785 	struct qbe_value rtfunc = mkrtfunc(ctx, "rt.abort");
    786 	if (expr->assert.cond) {
    787 		struct qbe_value bfailed = mklabel(ctx, &failedl, "failed.%d");
    788 		struct qbe_value bpassed = mklabel(ctx, &passedl, "passed.%d");
    789 		struct gen_value cond = gen_expr(ctx, expr->assert.cond);
    790 		struct qbe_value qcond = mkqval(ctx, &cond);
    791 		pushi(ctx->current, NULL, Q_JNZ, &qcond, &bpassed, &bfailed, NULL);
    792 		push(&ctx->current->body, &failedl);
    793 		msg = gen_expr(ctx, expr->assert.message);
    794 	} else {
    795 		msg = gen_expr(ctx, expr->assert.message);
    796 	}
    797 
    798 	struct qbe_value qmsg = mkqval(ctx, &msg);
    799 	pushi(ctx->current, NULL, Q_CALL, &rtfunc, &qmsg, NULL);
    800 
    801 	if (expr->assert.cond) {
    802 		push(&ctx->current->body, &passedl);
    803 	}
    804 
    805 	return gv_void;
    806 }
    807 
    808 static struct gen_value
    809 gen_expr_assign_slice(struct gen_context *ctx, const struct expression *expr)
    810 {
    811 	struct gen_value obj = gen_expr(ctx, expr->assign.object);
    812 	struct gen_value val = gen_expr(ctx, expr->assign.value);
    813 	struct qbe_value qobj = mkqval(ctx, &obj);
    814 	struct qbe_value qval = mkqval(ctx, &val);
    815 	struct qbe_value step = constl(ctx->arch.ptr->size);
    816 
    817 	struct qbe_value ptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    818 	struct qbe_value olen = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    819 	struct qbe_value vlen = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    820 
    821 	pushi(ctx->current, &ptr, Q_ADD, &qobj, &step, NULL);
    822 	pushi(ctx->current, &olen, Q_LOADL, &ptr, NULL);
    823 	pushi(ctx->current, &ptr, Q_ADD, &qval, &step, NULL);
    824 	pushi(ctx->current, &vlen, Q_LOADL, &ptr, NULL);
    825 
    826 	struct qbe_statement linvalid, lvalid;
    827 	struct qbe_value binvalid = mklabel(ctx, &linvalid, ".%d");
    828 	struct qbe_value bvalid = mklabel(ctx, &lvalid, ".%d");
    829 	struct qbe_value tmp = mkqtmp(ctx, &qbe_long, ".%d");
    830 	pushi(ctx->current, &tmp, Q_CEQL, &olen, &vlen, NULL);
    831 	pushi(ctx->current, NULL, Q_JNZ, &tmp, &bvalid, &binvalid, NULL);
    832 	push(&ctx->current->body, &linvalid);
    833 	gen_fixed_abort(ctx, expr->loc, ABORT_OOB);
    834 	push(&ctx->current->body, &lvalid);
    835 
    836 	struct qbe_value rtmemmove = mkrtfunc(ctx, "rt.memmove");
    837 	struct qbe_value optr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    838 	struct qbe_value vptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    839 	pushi(ctx->current, &optr, Q_LOADL, &qobj, NULL);
    840 	pushi(ctx->current, &vptr, Q_LOADL, &qval, NULL);
    841 	tmp = constl(expr->assign.object->result->array.members->size);
    842 	pushi(ctx->current, &olen, Q_MUL, &olen, &tmp, NULL);
    843 	pushi(ctx->current, NULL, Q_CALL, &rtmemmove, &optr, &vptr, &olen, NULL);
    844 
    845 	return gv_void;
    846 }
    847 
    848 static struct gen_value
    849 gen_expr_assign(struct gen_context *ctx, const struct expression *expr)
    850 {
    851 	struct expression *object = expr->assign.object;
    852 	struct expression *value = expr->assign.value;
    853 	if (object->type == EXPR_SLICE) {
    854 		return gen_expr_assign_slice(ctx, expr);
    855 	}
    856 	assert(object->type == EXPR_ACCESS || expr->assign.indirect); // Invariant
    857 
    858 	struct gen_value obj;
    859 	if (expr->assign.indirect) {
    860 		obj = gen_expr(ctx, object);
    861 		obj.type = type_dealias(object->result)->pointer.referent;
    862 	} else {
    863 		obj = gen_expr_access_addr(ctx, object);
    864 	}
    865 	if (expr->assign.op == BIN_LEQUAL) {
    866 		gen_expr_at(ctx, value, obj);
    867 	} else if (expr->assign.op == BIN_LAND || expr->assign.op == BIN_LOR) {
    868 		struct qbe_statement lrval, lshort;
    869 		struct qbe_value brval = mklabel(ctx, &lrval, ".%d");
    870 		struct qbe_value bshort = mklabel(ctx, &lshort, ".%d");
    871 		struct gen_value load = gen_load(ctx, obj);
    872 		struct qbe_value qload = mkqval(ctx, &load);
    873 		if (expr->binarithm.op == BIN_LAND) {
    874 			pushi(ctx->current, NULL, Q_JNZ, &qload, &brval,
    875 				&bshort, NULL);
    876 		} else {
    877 			pushi(ctx->current, NULL, Q_JNZ, &qload, &bshort,
    878 				&brval, NULL);
    879 		}
    880 		push(&ctx->current->body, &lrval);
    881 		gen_expr_at(ctx, value, obj);
    882 		if (!expr->binarithm.rvalue->terminates) {
    883 			pushi(ctx->current, NULL, Q_JMP, &bshort, NULL);
    884 		}
    885 		push(&ctx->current->body, &lshort);
    886 	} else {
    887 		struct gen_value lvalue = gen_load(ctx, obj);
    888 		struct gen_value rvalue = gen_expr(ctx, value);
    889 		struct qbe_value qlval = mkqval(ctx, &lvalue);
    890 		struct qbe_value qrval = mkqval(ctx, &rvalue);
    891 		enum qbe_instr instr = binarithm_for_op(ctx,
    892 			expr->assign.op, lvalue.type);
    893 		pushi(ctx->current, &qlval, instr, &qlval, &qrval, NULL);
    894 		gen_store(ctx, obj, lvalue);
    895 	}
    896 
    897 	return gv_void;
    898 }
    899 
    900 static struct qbe_value
    901 extend(struct gen_context *ctx, struct qbe_value v, const struct type *type)
    902 {
    903 	enum qbe_instr op;
    904 	switch (type->size) {
    905 	case 1:
    906 		op = type_is_signed(type) ? Q_EXTSB : Q_EXTUB;
    907 		break;
    908 	case 2:
    909 		op = type_is_signed(type) ? Q_EXTSH : Q_EXTUH;
    910 		break;
    911 	default:
    912 		return v;
    913 	}
    914 
    915 	struct qbe_value temp = mkqtmp(ctx, &qbe_word, "ext.%d");
    916 	pushi(ctx->current, &temp, op, &v, NULL);
    917 	return temp;
    918 }
    919 
    920 static struct gen_value
    921 gen_expr_binarithm(struct gen_context *ctx, const struct expression *expr)
    922 {
    923 	const struct type *ltype = type_dealias(expr->binarithm.lvalue->result);
    924 	const struct type *rtype = type_dealias(expr->binarithm.rvalue->result);
    925 	struct gen_value result = mkgtemp(ctx, expr->result, ".%d");
    926 	struct qbe_value qresult = mkqval(ctx, &result);
    927 
    928 	if (expr->binarithm.op == BIN_LAND || expr->binarithm.op == BIN_LOR) {
    929 		struct qbe_statement lrval, lshort;
    930 		struct qbe_value brval = mklabel(ctx, &lrval, ".%d");
    931 		struct qbe_value bshort = mklabel(ctx, &lshort, ".%d");
    932 		struct gen_value lval = gen_expr(ctx, expr->binarithm.lvalue);
    933 		struct qbe_value qlval = mkqval(ctx, &lval);
    934 		pushi(ctx->current, &qresult, Q_COPY, &qlval, NULL);
    935 		if (expr->binarithm.op == BIN_LAND) {
    936 			pushi(ctx->current, NULL, Q_JNZ, &qresult, &brval,
    937 				&bshort, NULL);
    938 		} else {
    939 			pushi(ctx->current, NULL, Q_JNZ, &qresult, &bshort,
    940 				&brval, NULL);
    941 		}
    942 		push(&ctx->current->body, &lrval);
    943 		struct gen_value rval = gen_expr(ctx, expr->binarithm.rvalue);
    944 		struct qbe_value qrval = mkqval(ctx, &rval);
    945 		pushi(ctx->current, &qresult, Q_COPY, &qrval, NULL);
    946 		if (!expr->binarithm.rvalue->terminates) {
    947 			pushi(ctx->current, NULL, Q_JMP, &bshort, NULL);
    948 		}
    949 		push(&ctx->current->body, &lshort);
    950 		return result;
    951 	}
    952 
    953 	struct gen_value lvalue = gen_expr(ctx, expr->binarithm.lvalue);
    954 	struct gen_value rvalue = gen_expr(ctx, expr->binarithm.rvalue);
    955 	struct qbe_value qlval = mkqval(ctx, &lvalue);
    956 	struct qbe_value qrval = mkqval(ctx, &rvalue);
    957 
    958 	switch (expr->binarithm.op) {
    959 	case BIN_GREATER:
    960 	case BIN_GREATEREQ:
    961 	case BIN_LEQUAL:
    962 	case BIN_LESS:
    963 	case BIN_LESSEQ:
    964 	case BIN_NEQUAL:
    965 		qlval = extend(ctx, qlval, ltype);
    966 		qrval = extend(ctx, qrval, rtype);
    967 		break;
    968 	default:
    969 		break;
    970 	}
    971 
    972 	assert((ltype->storage == STORAGE_STRING) == (rtype->storage == STORAGE_STRING));
    973 	if (ltype->storage == STORAGE_STRING) {
    974 		struct qbe_value rtfunc = mkrtfunc(ctx, "rt.strcmp");
    975 		pushi(ctx->current, &qresult, Q_CALL,
    976 			&rtfunc, &qlval, &qrval, NULL);
    977 		if (expr->binarithm.op == BIN_NEQUAL) {
    978 			struct qbe_value one = constl(1);
    979 			pushi(ctx->current, &qresult, Q_XOR, &qresult, &one, NULL);
    980 		} else {
    981 			assert(expr->binarithm.op == BIN_LEQUAL);
    982 		}
    983 		return result;
    984 	}
    985 	enum qbe_instr instr = binarithm_for_op(ctx, expr->binarithm.op,
    986 		expr->binarithm.lvalue->result);
    987 	pushi(ctx->current, &qresult, instr, &qlval, &qrval, NULL);
    988 	return result;
    989 }
    990 
    991 static void
    992 gen_expr_binding_unpack_static(struct gen_context *ctx,
    993 	const struct expression_binding *binding)
    994 {
    995 	assert(binding->object == NULL);
    996 
    997 	struct tuple_constant *tupleconst =
    998 		binding->initializer->constant.tuple;
    999 
   1000 	for (const struct binding_unpack *unpack = binding->unpack;
   1001 			unpack; unpack = unpack->next) {
   1002 		if (unpack->object == NULL) {
   1003 			goto done;
   1004 		}
   1005 		assert(unpack->object->otype == O_DECL);
   1006 
   1007 		struct declaration decl = {
   1008 			.type = DECL_GLOBAL,
   1009 			.ident = unpack->object->ident,
   1010 			.global = {
   1011 				.type = unpack->object->type,
   1012 				.value = tupleconst->value,
   1013 			},
   1014 		};
   1015 		gen_global_decl(ctx, &decl);
   1016 
   1017 done:
   1018 		tupleconst = tupleconst->next;
   1019 	}
   1020 }
   1021 
   1022 static void
   1023 gen_expr_binding_unpack(struct gen_context *ctx,
   1024 	const struct expression_binding *binding)
   1025 {
   1026 	assert(binding->object == NULL);
   1027 
   1028 	const struct type *type = binding->initializer->result;
   1029 	char *tuple_name = gen_name(ctx, "tupleunpack.%d");
   1030 	struct gen_value tuple_gv = {
   1031 		.kind = GV_TEMP,
   1032 		.type = type,
   1033 		.name = tuple_name,
   1034 	};
   1035 	struct qbe_value tuple_qv = mklval(ctx, &tuple_gv);
   1036 	struct qbe_value sz = constl(type->size);
   1037 	enum qbe_instr alloc = alloc_for_align(type->align);
   1038 	pushprei(ctx->current, &tuple_qv, alloc, &sz, NULL);
   1039 
   1040 	gen_expr_at(ctx, binding->initializer, tuple_gv);
   1041 
   1042 	for (const struct binding_unpack *unpack = binding->unpack;
   1043 			unpack; unpack = unpack->next) {
   1044 		if (unpack->object == NULL) {
   1045 			continue;
   1046 		}
   1047 		assert(unpack->object->otype != O_DECL);
   1048 
   1049 		const struct type *type = unpack->object->type;
   1050 		struct gen_value item_gv = {
   1051 			.kind = GV_TEMP,
   1052 			.type = type,
   1053 			.name = gen_name(ctx, "binding.%d"),
   1054 		};
   1055 		struct gen_binding *gb = xcalloc(1, sizeof(struct gen_binding));
   1056 		gb->value = item_gv;
   1057 		gb->object = unpack->object;
   1058 		gb->next = ctx->bindings;
   1059 		ctx->bindings = gb;
   1060 		struct qbe_value item_qv = mklval(ctx, &gb->value);
   1061 		struct qbe_value offs = constl(unpack->offset);
   1062 		pushprei(ctx->current, &item_qv, Q_ADD, &tuple_qv, &offs, NULL);
   1063 	}
   1064 }
   1065 
   1066 static struct gen_value
   1067 gen_expr_binding(struct gen_context *ctx, const struct expression *expr)
   1068 {
   1069 	for (const struct expression_binding *binding = &expr->binding;
   1070 			binding; binding = binding->next) {
   1071 		if (binding->unpack) {
   1072 			if (binding->unpack->object->otype == O_DECL) {
   1073 				gen_expr_binding_unpack_static(ctx, binding);
   1074 			} else {
   1075 				gen_expr_binding_unpack(ctx, binding);
   1076 			}
   1077 			continue;
   1078 		}
   1079 
   1080 		if (binding->object->otype == O_DECL) {
   1081 			// static binding
   1082 			struct declaration decl = {
   1083 				.type = DECL_GLOBAL,
   1084 				.ident = binding->object->ident,
   1085 				.global = {
   1086 					.type = binding->object->type,
   1087 					.value = binding->initializer,
   1088 				},
   1089 			};
   1090 			gen_global_decl(ctx, &decl);
   1091 			continue;
   1092 		}
   1093 
   1094 		const struct type *type = binding->initializer->result;
   1095 		struct gen_binding *gb = xcalloc(1, sizeof(struct gen_binding));
   1096 		gb->value.kind = GV_TEMP;
   1097 		gb->value.type = type;
   1098 		gb->value.name = gen_name(ctx, "binding.%d");
   1099 		gb->object = binding->object;
   1100 		gb->next = ctx->bindings;
   1101 		ctx->bindings = gb;
   1102 
   1103 		struct qbe_value qv = mklval(ctx, &gb->value);
   1104 		struct qbe_value sz = constl(type->size);
   1105 		enum qbe_instr alloc = alloc_for_align(type->align);
   1106 		pushprei(ctx->current, &qv, alloc, &sz, NULL);
   1107 		gen_expr_at(ctx, binding->initializer, gb->value);
   1108 	}
   1109 	return gv_void;
   1110 }
   1111 
   1112 static struct gen_value
   1113 gen_expr_control(struct gen_context *ctx, const struct expression *expr)
   1114 {
   1115 	struct gen_scope *scope = gen_scope_lookup(ctx, expr->control.scope);
   1116 
   1117 	if (expr->control.value) {
   1118 		struct gen_value result = gen_expr_with(ctx,
   1119 			expr->control.value, scope->out);
   1120 		branch_copyresult(ctx, result,
   1121 			scope->result, scope->out);
   1122 	}
   1123 
   1124 	struct gen_scope *deferred = ctx->scope;
   1125 	while (deferred != NULL) {
   1126 		gen_defers(ctx, deferred);
   1127 		if (deferred == scope) {
   1128 			break;
   1129 		}
   1130 		deferred = deferred->parent;
   1131 	}
   1132 
   1133 	switch (expr->type) {
   1134 	case EXPR_BREAK:
   1135 		assert(scope->scope->class == SCOPE_LOOP);
   1136 		pushi(ctx->current, NULL, Q_JMP, scope->end, NULL);
   1137 		break;
   1138 	case EXPR_CONTINUE:
   1139 		assert(scope->scope->class == SCOPE_LOOP);
   1140 		pushi(ctx->current, NULL, Q_JMP, scope->after, NULL);
   1141 		break;
   1142 	case EXPR_YIELD:
   1143 		assert(scope->scope->class == SCOPE_COMPOUND);
   1144 		pushi(ctx->current, NULL, Q_JMP, scope->end, NULL);
   1145 		break;
   1146 	default: abort(); // Invariant
   1147 	}
   1148 	return gv_void;
   1149 }
   1150 
   1151 static struct gen_value
   1152 gen_expr_call(struct gen_context *ctx, const struct expression *expr)
   1153 {
   1154 	struct gen_value lvalue = gen_expr(ctx, expr->call.lvalue);
   1155 	lvalue = gen_autoderef(ctx, lvalue);
   1156 
   1157 	const struct type *rtype = type_dealias(lvalue.type);
   1158 	assert(rtype->storage == STORAGE_FUNCTION);
   1159 
   1160 	if (rtype->func.flags & FN_NORETURN) {
   1161 		for (struct gen_scope *scope = ctx->scope; scope;
   1162 				scope = scope->parent) {
   1163 			gen_defers(ctx, scope);
   1164 		}
   1165 	}
   1166 
   1167 	struct qbe_statement call = {
   1168 		.type = Q_INSTR,
   1169 		.instr = Q_CALL,
   1170 	};
   1171 	struct gen_value rval = gv_void;
   1172 	if (type_dealias(rtype->func.result)->storage != STORAGE_VOID) {
   1173 		rval = mkgtemp(ctx, rtype->func.result, "returns.%d");
   1174 		call.out = xcalloc(1, sizeof(struct qbe_value));
   1175 		*call.out = mkqval(ctx, &rval);
   1176 		call.out->type = qtype_lookup(ctx, rtype->func.result, false);
   1177 	}
   1178 
   1179 	bool cvar = false;
   1180 	struct type_func_param *param = rtype->func.params;
   1181 	struct qbe_arguments *args, **next = &call.args;
   1182 	args = *next = xcalloc(1, sizeof(struct qbe_arguments));
   1183 	args->value = mkqval(ctx, &lvalue);
   1184 	next = &args->next;
   1185 	for (struct call_argument *carg = expr->call.args;
   1186 			carg; carg = carg->next) {
   1187 		args = *next = xcalloc(1, sizeof(struct qbe_arguments));
   1188 		struct gen_value arg = gen_expr(ctx, carg->value);
   1189 		args->value = mkqval(ctx, &arg);
   1190 		args->value.type = qtype_lookup(ctx, carg->value->result, false);
   1191 		next = &args->next;
   1192 		if (param) {
   1193 			param = param->next;
   1194 		}
   1195 		if (!param && !cvar && rtype->func.variadism == VARIADISM_C) {
   1196 			cvar = true;
   1197 			args = *next = xcalloc(1, sizeof(struct qbe_arguments));
   1198 			args->value.kind = QV_VARIADIC;
   1199 			next = &args->next;
   1200 		}
   1201 	}
   1202 	push(&ctx->current->body, &call);
   1203 
   1204 	return rval;
   1205 }
   1206 
   1207 static struct gen_value gen_expr_cast(struct gen_context *ctx,
   1208 		const struct expression *expr);
   1209 
   1210 static struct gen_value gen_subset_match_tests(struct gen_context *ctx,
   1211 	struct qbe_value bmatch, struct qbe_value bnext,
   1212 	struct qbe_value tag, const struct type *type);
   1213 
   1214 static struct gen_value gen_nested_match_tests(struct gen_context *ctx,
   1215 		struct gen_value object, struct qbe_value bmatch,
   1216 		struct qbe_value bnext, struct qbe_value tag,
   1217 		const struct type *type);
   1218 
   1219 static struct gen_value
   1220 gen_type_assertion_or_test(struct gen_context *ctx, const struct expression *expr,
   1221 		struct gen_value base)
   1222 {
   1223 	assert(expr->cast.kind == C_TEST || expr->cast.kind == C_ASSERTION);
   1224 	const struct type *want = expr->cast.secondary;
   1225 	struct qbe_value tag = mkqtmp(ctx,
   1226 		qtype_lookup(ctx, &builtin_type_uint, false), ".%d");
   1227 	enum qbe_instr load = load_for_type(ctx, &builtin_type_uint);
   1228 	struct qbe_value qbase = mkqval(ctx, &base);
   1229 	pushi(ctx->current, &tag, load, &qbase, NULL);
   1230 
   1231 	struct qbe_statement failedl, passedl;
   1232 	struct qbe_value bfailed, bpassed = mklabel(ctx, &passedl, "passed.%d");
   1233 	if (expr->cast.kind == C_ASSERTION) {
   1234 		bfailed = mklabel(ctx, &failedl, "failed.%d");
   1235 	} else {
   1236 		bfailed = bpassed;
   1237 	}
   1238 	struct gen_value result = {0};
   1239 	if (tagged_select_subtype(expr->cast.value->result, want)) {
   1240 		result = gen_nested_match_tests(ctx, base, bpassed,
   1241 				bfailed, tag, want);
   1242 	} else if (tagged_subset_compat(expr->cast.value->result, want)) {
   1243 		result = gen_subset_match_tests(ctx, bpassed, bfailed, tag,
   1244 				type_dealias(want));
   1245 	} else {
   1246 		abort();
   1247 	}
   1248 
   1249 	if (expr->cast.kind == C_ASSERTION) {
   1250 		push(&ctx->current->body, &failedl);
   1251 		gen_fixed_abort(ctx, expr->loc, ABORT_TYPE_ASSERTION);
   1252 	}
   1253 	push(&ctx->current->body, &passedl);
   1254 	return result;
   1255 }
   1256 
   1257 static void
   1258 gen_expr_cast_slice_at(struct gen_context *ctx,
   1259 	const struct expression *expr, struct gen_value out)
   1260 {
   1261 	const struct type *to = expr->result,
   1262 		*from = type_dealias(expr->cast.value->result);
   1263 	if (from->storage == STORAGE_POINTER) {
   1264 		from = type_dealias(from->pointer.referent);
   1265 	}
   1266 	assert(from->storage == STORAGE_ARRAY);
   1267 	assert(from->array.length != SIZE_UNDEFINED);
   1268 
   1269 	enum qbe_instr store = store_for_type(ctx, &builtin_type_size);
   1270 	struct qbe_value base = mklval(ctx, &out);
   1271 	struct qbe_value sz = constl(to->size);
   1272 	if (from->array.length == 0) {
   1273 		struct qbe_value tmp = constl(0);
   1274 		pushi(ctx->current, NULL, store, &tmp, &base, NULL);
   1275 	} else {
   1276 		struct gen_value value = gen_expr(ctx, expr->cast.value);
   1277 		struct qbe_value qvalue = mkqval(ctx, &value);
   1278 		pushi(ctx->current, NULL, store, &qvalue, &base, NULL);
   1279 	}
   1280 	struct qbe_value qptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   1281 	sz = constl(builtin_type_size.size);
   1282 	struct qbe_value ln = constl(from->array.length);
   1283 	pushi(ctx->current, &qptr, Q_ADD, &base, &sz, NULL);
   1284 	pushi(ctx->current, NULL, store, &ln, &qptr, NULL);
   1285 	pushi(ctx->current, &qptr, Q_ADD, &qptr, &sz, NULL);
   1286 	pushi(ctx->current, NULL, store, &ln, &qptr, NULL);
   1287 }
   1288 
   1289 // Returns true if object's storage can be interpreted as want.
   1290 static bool
   1291 tagged_align_compat(const struct type *object, const struct type *want)
   1292 {
   1293 	assert(type_dealias(object)->storage == STORAGE_TAGGED);
   1294 	assert(type_dealias(want)->storage == STORAGE_TAGGED);
   1295 	return object->align == want->align
   1296 		|| want->size == builtin_type_uint.size;
   1297 }
   1298 
   1299 static void
   1300 gen_expr_cast_tagged_at(struct gen_context *ctx,
   1301 	const struct expression *expr, struct gen_value out)
   1302 {
   1303 	assert(expr->type == EXPR_CAST);
   1304 	const struct type *to = expr->result, *from = expr->cast.value->result;
   1305 	const struct type *subtype = tagged_select_subtype(to, from);
   1306 
   1307 	if (!subtype && tagged_align_compat(from, to)) {
   1308 		// Case 1: from is a union whose members are a subset of to, and
   1309 		// the alignment matches, so we can just interpret values of
   1310 		// type 'from' as if it were of type 'to'
   1311 		struct gen_value out2 = out;
   1312 		out2.type = from;
   1313 		gen_expr_at(ctx, expr->cast.value, out2);
   1314 		if (expr->cast.kind == C_ASSERTION) {
   1315 			gen_type_assertion_or_test(ctx, expr, out2);
   1316 		}
   1317 	} else if (!subtype) {
   1318 		// Case 2: like case 1, but with an alignment mismatch; more
   1319 		// work is required.
   1320 		struct gen_value value = gen_expr(ctx, expr->cast.value);
   1321 		struct qbe_value qval = mkqval(ctx, &value);
   1322 		if (expr->cast.kind == C_ASSERTION) {
   1323 			gen_type_assertion_or_test(ctx, expr, value);
   1324 		}
   1325 		struct qbe_value qout = mkqval(ctx, &out);
   1326 		struct qbe_value tag = mkqtmp(ctx,
   1327 			qtype_lookup(ctx, &builtin_type_uint, false), "tag.%d");
   1328 		enum qbe_instr load = load_for_type(ctx, &builtin_type_uint);
   1329 		enum qbe_instr store = store_for_type(ctx, &builtin_type_uint);
   1330 		pushi(ctx->current, &tag, load, &qval, NULL);
   1331 		pushi(ctx->current, NULL, store, &tag, &qout, NULL);
   1332 		if (to->size == builtin_type_uint.size ||
   1333 				from->size == builtin_type_uint.size) {
   1334 			// No data area to copy
   1335 			return;
   1336 		}
   1337 
   1338 		subtype = tagged_subset_compat(to, from) ? from : to;
   1339 		const struct type *innertype = type_store_tagged_to_union(
   1340 				ctx->store, type_dealias(subtype));
   1341 		struct gen_value iout = mkgtemp(ctx, innertype, ".%d");
   1342 		struct gen_value ival = mkgtemp(ctx, innertype, ".%d");
   1343 		struct qbe_value qiout = mkqval(ctx, &iout);
   1344 		struct qbe_value qival = mkqval(ctx, &ival);
   1345 		struct qbe_value offs = constl(to->align);
   1346 		pushi(ctx->current, &qiout, Q_ADD, &qout, &offs, NULL);
   1347 		offs = constl(from->align);
   1348 		pushi(ctx->current, &qival, Q_ADD, &qval, &offs, NULL);
   1349 		gen_copy_aligned(ctx, iout, ival);
   1350 	} else {
   1351 		// Case 3: from is a member of to
   1352 		assert(subtype == from); // Lowered by check
   1353 		struct qbe_value qout = mkqval(ctx, &out);
   1354 		struct qbe_value id = constw(subtype->id);
   1355 		enum qbe_instr store = store_for_type(ctx, &builtin_type_uint);
   1356 		pushi(ctx->current, NULL, store, &id, &qout, NULL);
   1357 		if (subtype->size == 0) {
   1358 			gen_expr(ctx, expr->cast.value); // side-effects
   1359 			return;
   1360 		}
   1361 
   1362 		struct gen_value storage = mkgtemp(ctx, subtype, ".%d");
   1363 		struct qbe_value qstor = mklval(ctx, &storage);
   1364 		struct qbe_value offs = constl(to->align);
   1365 		pushi(ctx->current, &qstor, Q_ADD, &qout, &offs, NULL);
   1366 		gen_expr_at(ctx, expr->cast.value, storage);
   1367 	}
   1368 }
   1369 
   1370 static bool
   1371 cast_prefers_at(const struct expression *expr)
   1372 {
   1373 	const struct type *to = expr->result, *from = expr->cast.value->result;
   1374 	if (expr->cast.kind == C_TEST) {
   1375 		return false;
   1376 	}
   1377 	// tagged => *; subtype compatible
   1378 	if (type_dealias(from)->storage == STORAGE_TAGGED
   1379 			&& tagged_select_subtype(from, to)) {
   1380 		return false;
   1381 	}
   1382 	// * => tagged
   1383 	if (type_dealias(to)->storage == STORAGE_TAGGED) {
   1384 		return true;
   1385 	}
   1386 	// array => array
   1387 	if (type_dealias(to)->storage == STORAGE_ARRAY
   1388 			&& type_dealias(from)->storage == STORAGE_ARRAY) {
   1389 		return true;
   1390 	}
   1391 	// array => slice
   1392 	if (type_dealias(to)->storage == STORAGE_SLICE) {
   1393 		switch (type_dealias(from)->storage) {
   1394 		case STORAGE_ARRAY:
   1395 			return true;
   1396 		case STORAGE_POINTER:
   1397 			from = type_dealias(from)->pointer.referent;
   1398 			return type_dealias(from)->storage == STORAGE_ARRAY;
   1399 		default:
   1400 			return false;
   1401 		}
   1402 	}
   1403 	return false;
   1404 }
   1405 
   1406 static void
   1407 gen_expr_cast_array_at(struct gen_context *ctx,
   1408 	const struct expression *expr, struct gen_value out)
   1409 {
   1410 	const struct type *typeout = type_dealias(expr->result);
   1411 	const struct type *typein = type_dealias(expr->cast.value->result);
   1412 	gen_expr_at(ctx, expr->cast.value, out);
   1413 	if (!typein->array.expandable) {
   1414 		return;
   1415 	}
   1416 
   1417 	assert(typein->array.length != SIZE_UNDEFINED
   1418 			&& typeout->array.length != SIZE_UNDEFINED);
   1419 	assert(typeout->array.length >= typein->array.length);
   1420 
   1421 	const struct type *membtype = typein->array.members;
   1422 	size_t remain = typeout->array.length - typein->array.length;
   1423 
   1424 	struct qbe_value base = mkqval(ctx, &out);
   1425 	struct qbe_value offs = constl((typein->array.length - 1) * membtype->size);
   1426 	struct gen_value next = mkgtemp(ctx, membtype, ".%d");
   1427 	struct qbe_value ptr = mklval(ctx, &next);
   1428 	struct gen_value item = mkgtemp(ctx, membtype, "item.%d");
   1429 	struct qbe_value qitem = mklval(ctx, &item);
   1430 	pushi(ctx->current, &qitem, Q_ADD, &base, &offs, NULL);
   1431 
   1432 	if (remain * membtype->size <= 128) {
   1433 		struct gen_value last = gen_load(ctx, item);
   1434 		for (size_t n = typein->array.length; n < typeout->array.length; ++n) {
   1435 			struct qbe_value offs = constl(n * membtype->size);
   1436 			pushi(ctx->current, &ptr, Q_ADD, &base, &offs, NULL);
   1437 			gen_store(ctx, next, last);
   1438 		}
   1439 		return;
   1440 	}
   1441 
   1442 	offs = constl(typein->array.length * membtype->size);
   1443 	pushi(ctx->current, &ptr, Q_ADD, &base, &offs, NULL);
   1444 
   1445 	struct qbe_value rtfunc = mkrtfunc(ctx, "rt.memcpy");
   1446 	struct qbe_value dtemp = mklval(ctx, &next);
   1447 	struct qbe_value stemp = mklval(ctx, &item);
   1448 	struct qbe_value sz = constl(remain * membtype->size);
   1449 	pushi(ctx->current, NULL, Q_CALL, &rtfunc, &dtemp, &stemp, &sz, NULL);
   1450 }
   1451 
   1452 static void
   1453 gen_expr_cast_at(struct gen_context *ctx,
   1454 	const struct expression *expr, struct gen_value out)
   1455 {
   1456 	if (!cast_prefers_at(expr)) {
   1457 		struct gen_value result = gen_expr_cast(ctx, expr);
   1458 		if (!expr->terminates) {
   1459 			gen_store(ctx, out, result);
   1460 		}
   1461 		return;
   1462 	}
   1463 
   1464 	if (expr->cast.lowered) {
   1465 		pushc(ctx->current, "gen lowered cast");
   1466 	}
   1467 
   1468 	const struct type *to = expr->result;
   1469 	switch (type_dealias(to)->storage) {
   1470 	case STORAGE_SLICE:
   1471 		gen_expr_cast_slice_at(ctx, expr, out);
   1472 		break;
   1473 	case STORAGE_TAGGED:
   1474 		gen_expr_cast_tagged_at(ctx, expr, out);
   1475 		break;
   1476 	case STORAGE_ARRAY:
   1477 		gen_expr_cast_array_at(ctx, expr, out);
   1478 		break;
   1479 	default: abort(); // Invariant
   1480 	}
   1481 }
   1482 
   1483 static struct qbe_value nested_tagged_offset(const struct type *tu,
   1484 		const struct type *targed);
   1485 
   1486 static struct gen_value
   1487 gen_expr_cast(struct gen_context *ctx, const struct expression *expr)
   1488 {
   1489 	const struct type *to = expr->cast.secondary,
   1490 		*from = expr->cast.value->result;
   1491 	if (expr->cast.kind != C_CAST) {
   1492 		bool is_valid_tagged, is_valid_pointer;
   1493 		is_valid_tagged = type_dealias(from)->storage == STORAGE_TAGGED
   1494 				&& (tagged_select_subtype(from, to)
   1495 				|| tagged_subset_compat(from, to));
   1496 		is_valid_pointer = type_dealias(from)->storage == STORAGE_POINTER
   1497 				&& (type_dealias(to)->storage == STORAGE_POINTER
   1498 				|| type_dealias(to)->storage == STORAGE_NULL);
   1499 		assert(is_valid_tagged || is_valid_pointer);
   1500 		if (expr->cast.kind == C_TEST && is_valid_tagged) {
   1501 			return gen_type_assertion_or_test(ctx, expr,
   1502 					gen_expr(ctx, expr->cast.value));
   1503 		}
   1504 	}
   1505 
   1506 	if (cast_prefers_at(expr)) {
   1507 		struct gen_value out = mkgtemp(ctx, expr->result, "object.%d");
   1508 		struct qbe_value base = mkqval(ctx, &out);
   1509 		struct qbe_value sz = constl(expr->result->size);
   1510 		enum qbe_instr alloc = alloc_for_align(expr->result->align);
   1511 		pushprei(ctx->current, &base, alloc, &sz, NULL);
   1512 		gen_expr_cast_at(ctx, expr, out);
   1513 		return out;
   1514 	}
   1515 
   1516 	if (expr->cast.lowered) {
   1517 		pushc(ctx->current, "gen lowered cast");
   1518 	}
   1519 
   1520 	// Special cases
   1521 	bool want_null = false;
   1522 	switch (type_dealias(to)->storage) {
   1523 	case STORAGE_NULL:
   1524 		want_null = true;
   1525 		// fallthrough
   1526 	case STORAGE_POINTER:
   1527 		if (type_dealias(from)->storage == STORAGE_SLICE) {
   1528 			struct gen_value value = gen_expr(ctx, expr->cast.value);
   1529 			value.type = to;
   1530 			return gen_load(ctx, value);
   1531 		}
   1532 		if (type_dealias(from)->storage != STORAGE_POINTER) {
   1533 			break;
   1534 		}
   1535 
   1536 		struct gen_value val = gen_expr(ctx, expr->cast.value);
   1537 		struct qbe_value qval = mkqval(ctx, &val);
   1538 		if (expr->cast.kind == C_TEST) {
   1539 			struct gen_value out =
   1540 				mkgtemp(ctx, &builtin_type_bool, ".%d");
   1541 			struct qbe_value qout = mkqval(ctx, &out);
   1542 			struct qbe_value zero = constl(0);
   1543 
   1544 			enum qbe_instr compare = want_null? Q_CEQL : Q_CNEL;
   1545 			pushi(ctx->current, &qout, compare, &qval, &zero, NULL);
   1546 			return out;
   1547 		} else if (expr->cast.kind == C_ASSERTION) {
   1548 			struct qbe_statement failedl, passedl;
   1549 			struct qbe_value bfailed =
   1550 				mklabel(ctx, &failedl, "failed.%d");
   1551 			struct qbe_value bpassed =
   1552 				mklabel(ctx, &passedl, "passed.%d");
   1553 
   1554 			if (want_null) {
   1555 				pushi(ctx->current, NULL, Q_JNZ, &qval,
   1556 					&bfailed, &bpassed, NULL);
   1557 			} else {
   1558 				pushi(ctx->current, NULL, Q_JNZ, &qval,
   1559 					&bpassed, &bfailed, NULL);
   1560 			}
   1561 			push(&ctx->current->body, &failedl);
   1562 			gen_fixed_abort(ctx, expr->loc, ABORT_TYPE_ASSERTION);
   1563 
   1564 			push(&ctx->current->body, &passedl);
   1565 			if (want_null) {
   1566 				return (struct gen_value){
   1567 					.kind = GV_CONST,
   1568 					.type = &builtin_type_null,
   1569 					.lval = 0,
   1570 				};
   1571 			}
   1572 		}
   1573 		val.type = to;
   1574 		return val;
   1575 	case STORAGE_VOID:
   1576 		gen_expr(ctx, expr->cast.value); // Side-effects
   1577 		return gv_void;
   1578 	default: break;
   1579 	}
   1580 
   1581 	// Special case: tagged => non-tagged
   1582 	if (type_dealias(from)->storage == STORAGE_TAGGED) {
   1583 		struct gen_value value = gen_expr(ctx, expr->cast.value);
   1584 		struct qbe_value base = mkcopy(ctx, &value, ".%d");
   1585 		if (expr->cast.kind == C_ASSERTION) {
   1586 			gen_type_assertion_or_test(ctx, expr, value);
   1587 		}
   1588 
   1589 		struct qbe_value align = nested_tagged_offset(
   1590 				expr->cast.value->result, expr->cast.secondary);
   1591 		pushi(ctx->current, &base, Q_ADD, &base, &align, NULL);
   1592 		struct gen_value storage = (struct gen_value){
   1593 			.kind = GV_TEMP,
   1594 			.type = to,
   1595 			.name = base.name,
   1596 		};
   1597 		return gen_load(ctx, storage);
   1598 	}
   1599 
   1600 	// Special case: no conversion required
   1601 	if (type_dealias(to)->storage == type_dealias(from)->storage
   1602 			&& to->size == from->size) {
   1603 		struct gen_value value = gen_expr(ctx, expr->cast.value);
   1604 		value.type = to;
   1605 		return value;
   1606 	}
   1607 
   1608 	struct gen_value value = gen_expr(ctx, expr->cast.value);
   1609 	struct qbe_value qvalue = mkqval(ctx, &value);
   1610 	struct gen_value result = mkgtemp(ctx, expr->result, "cast.%d");
   1611 	struct qbe_value qresult = mkqval(ctx, &result);
   1612 	struct gen_value intermediate;
   1613 	struct qbe_value qintermediate;
   1614 
   1615 	from = lower_const(from, NULL);
   1616 
   1617 	enum qbe_instr op;
   1618 	bool is_signed = type_is_signed(from);
   1619 	enum type_storage fstor = type_dealias(from)->storage,
   1620 		tstor = type_dealias(to)->storage;
   1621 	switch (tstor) {
   1622 	case STORAGE_CHAR:
   1623 	case STORAGE_ENUM:
   1624 	case STORAGE_U8:
   1625 	case STORAGE_I8:
   1626 	case STORAGE_I16:
   1627 	case STORAGE_U16:
   1628 	case STORAGE_I32:
   1629 	case STORAGE_U32:
   1630 	case STORAGE_INT:
   1631 	case STORAGE_UINT:
   1632 	case STORAGE_I64:
   1633 	case STORAGE_U64:
   1634 	case STORAGE_UINTPTR:
   1635 	case STORAGE_RUNE:
   1636 	case STORAGE_SIZE:
   1637 		if (type_is_integer(from) && to->size <= from->size) {
   1638 			op = Q_COPY;
   1639 		} else if (type_is_integer(from) && to->size > from->size) {
   1640 			switch (from->size) {
   1641 			case 4: op = is_signed ? Q_EXTSW : Q_EXTUW; break;
   1642 			case 2: op = is_signed ? Q_EXTSH : Q_EXTUH; break;
   1643 			case 1: op = is_signed ? Q_EXTSB : Q_EXTUB; break;
   1644 			default: abort(); // Invariant
   1645 			}
   1646 		} else if (fstor == STORAGE_POINTER || fstor == STORAGE_NULL) {
   1647 			assert(tstor == STORAGE_UINTPTR);
   1648 			op = Q_COPY;
   1649 		} else if (fstor == STORAGE_RUNE) {
   1650 			op = Q_COPY;
   1651 		} else if (type_is_float(from)) {
   1652 			if (type_is_signed(to)) {
   1653 				switch (fstor) {
   1654 				case STORAGE_F32: op = Q_STOSI; break;
   1655 				case STORAGE_F64: op = Q_DTOSI; break;
   1656 				default: abort(); // Invariant
   1657 				}
   1658 			} else {
   1659 				switch (fstor) {
   1660 				case STORAGE_F32: op = Q_STOUI; break;
   1661 				case STORAGE_F64: op = Q_DTOUI; break;
   1662 				default: abort(); // Invariant
   1663 				}
   1664 			}
   1665 		} else {
   1666 			abort(); // Invariant
   1667 		}
   1668 		pushi(ctx->current, &qresult, op, &qvalue, NULL);
   1669 		break;
   1670 	case STORAGE_F32:
   1671 	case STORAGE_F64:
   1672 		if (type_is_float(from) && from->size == to->size) {
   1673 			op = Q_COPY;
   1674 		} else if (type_is_float(from) && to->size < from->size) {
   1675 			op = Q_TRUNCD;
   1676 		} else if (type_is_float(from) && to->size > from->size) {
   1677 			op = Q_EXTS;
   1678 		} else if (type_is_integer(from)) {
   1679 			if (type_is_signed(from)) {
   1680 				switch (from->size) {
   1681 				case 1:
   1682 				case 2:
   1683 					intermediate = mkgtemp(ctx,
   1684 						&builtin_type_i32, "cast.%d");
   1685 					qintermediate = mkqval(ctx, &intermediate);
   1686 					pushi(ctx->current, &qintermediate,
   1687 						from->size == 1? Q_EXTSB : Q_EXTSH,
   1688 						&qvalue, NULL);
   1689 					qvalue = qintermediate;
   1690 					/* fallthrough */
   1691 				case 4:
   1692 					op = Q_SWTOF;
   1693 					break;
   1694 				case 8:
   1695 					op = Q_SLTOF;
   1696 					break;
   1697 				default: abort(); // Invariant
   1698 				}
   1699 			} else {
   1700 				switch (from->size) {
   1701 				case 1:
   1702 				case 2:
   1703 					intermediate = mkgtemp(ctx,
   1704 						&builtin_type_i32, "cast.%d");
   1705 					qintermediate = mkqval(ctx, &intermediate);
   1706 					pushi(ctx->current, &qintermediate,
   1707 						from->size == 1? Q_EXTUB : Q_EXTUH,
   1708 						&qvalue, NULL);
   1709 					qvalue = qintermediate;
   1710 					/* fallthrough */
   1711 				case 4:
   1712 					op = Q_UWTOF;
   1713 					break;
   1714 				case 8:
   1715 					op = Q_ULTOF;
   1716 					break;
   1717 				default: abort(); // Invariant
   1718 				}
   1719 			}
   1720 		} else {
   1721 			abort(); // Invariant
   1722 		}
   1723 		pushi(ctx->current, &qresult, op, &qvalue, NULL);
   1724 		break;
   1725 	case STORAGE_NULL:
   1726 	case STORAGE_POINTER:
   1727 		pushi(ctx->current, &qresult, Q_COPY, &qvalue, NULL);
   1728 		break;
   1729 	case STORAGE_ARRAY:
   1730 		assert(from->storage == STORAGE_ARRAY);
   1731 		pushi(ctx->current, &qresult, Q_COPY, &qvalue, NULL);
   1732 		break;
   1733 	case STORAGE_SLICE:
   1734 		assert(from->storage == STORAGE_SLICE);
   1735 		pushi(ctx->current, &qresult, Q_COPY, &qvalue, NULL);
   1736 		break;
   1737 	case STORAGE_ALIAS:
   1738 	case STORAGE_BOOL:
   1739 	case STORAGE_FCONST:
   1740 	case STORAGE_FUNCTION:
   1741 	case STORAGE_ICONST:
   1742 	case STORAGE_RCONST:
   1743 	case STORAGE_STRING:
   1744 	case STORAGE_STRUCT:
   1745 	case STORAGE_TAGGED:
   1746 	case STORAGE_TUPLE:
   1747 	case STORAGE_UNION:
   1748 	case STORAGE_VALIST:
   1749 	case STORAGE_VOID:
   1750 		abort(); // Invariant
   1751 	}
   1752 
   1753 	return result;
   1754 }
   1755 
   1756 static struct gen_value
   1757 gen_expr_compound_with(struct gen_context *ctx,
   1758 	const struct expression *expr,
   1759 	struct gen_value *out)
   1760 {
   1761 	struct qbe_statement lend;
   1762 	struct qbe_value bend = mklabel(ctx, &lend, ".%d");
   1763 	struct gen_scope *scope = push_scope(ctx, expr->compound.scope);
   1764 	scope->end = &bend;
   1765 
   1766 	struct gen_value gvout = gv_void;
   1767 	if (!out) {
   1768 		gvout = mkgtemp(ctx, expr->result, ".%d");
   1769 	}
   1770 	scope->out = out;
   1771 	scope->result = gvout;
   1772 
   1773 	const struct expressions *exprs;
   1774 	for (exprs = &expr->compound.exprs; exprs->next; exprs = exprs->next) {
   1775 		gen_expr(ctx, exprs->expr);
   1776 	}
   1777 
   1778 	struct gen_value last = gen_expr_with(ctx, exprs->expr, out);
   1779 	branch_copyresult(ctx, last, gvout, out);
   1780 	pop_scope(ctx, !exprs->expr->terminates);
   1781 	push(&ctx->current->body, &lend);
   1782 	return gvout;
   1783 }
   1784 
   1785 static void
   1786 gen_const_array_at(struct gen_context *ctx,
   1787 	const struct expression *expr,
   1788 	struct gen_value out)
   1789 {
   1790 	struct array_constant *aexpr = expr->constant.array;
   1791 	struct qbe_value base = mkqval(ctx, &out);
   1792 
   1793 	size_t n = 0;
   1794 	const struct type *atype = type_dealias(expr->result);
   1795 	struct gen_value item = mkgtemp(ctx, atype->array.members, "item.%d");
   1796 	for (const struct array_constant *ac = aexpr; ac; ac = ac->next) {
   1797 		struct qbe_value offs = constl(n * atype->array.members->size);
   1798 		struct qbe_value ptr = mklval(ctx, &item);
   1799 		pushi(ctx->current, &ptr, Q_ADD, &base, &offs, NULL);
   1800 		gen_expr_at(ctx, ac->value, item);
   1801 		++n;
   1802 	}
   1803 
   1804 	assert(n == atype->array.length);
   1805 }
   1806 
   1807 static void
   1808 gen_const_string_at(struct gen_context *ctx,
   1809 	const struct expression *expr, struct gen_value out)
   1810 {
   1811 	const struct expression_constant *constexpr = &expr->constant;
   1812 	const char *val = constexpr->string.value;
   1813 	size_t len = constexpr->string.len;
   1814 
   1815 	// TODO: Generate string data structure as global also?
   1816 	struct qbe_value global = mkqtmp(ctx, ctx->arch.ptr, "strdata.%d");
   1817 	global.kind = QV_GLOBAL;
   1818 
   1819 	struct qbe_def *def = xcalloc(1, sizeof(struct qbe_def));
   1820 	def->name = global.name;
   1821 	def->kind = Q_DATA;
   1822 	def->data.align = ALIGN_UNDEFINED;
   1823 	def->data.items.type = QD_STRING;
   1824 	def->data.items.str = xcalloc(1, len);
   1825 	memcpy(def->data.items.str, val, len);
   1826 	def->data.items.sz = len;
   1827 
   1828 	if (len != 0) {
   1829 		qbe_append_def(ctx->out, def);
   1830 	} else {
   1831 		free(def);
   1832 		global = constl(0);
   1833 	}
   1834 
   1835 	enum qbe_instr store = store_for_type(ctx, &builtin_type_size);
   1836 	struct qbe_value strp = mkcopy(ctx, &out, ".%d");
   1837 	struct qbe_value qlen = constl(len);
   1838 	struct qbe_value offs = constl(builtin_type_size.size);
   1839 	pushi(ctx->current, NULL, store, &global, &strp, NULL);
   1840 	pushi(ctx->current, &strp, Q_ADD, &strp, &offs, NULL);
   1841 	pushi(ctx->current, NULL, store, &qlen, &strp, NULL);
   1842 	pushi(ctx->current, &strp, Q_ADD, &strp, &offs, NULL);
   1843 	pushi(ctx->current, NULL, store, &qlen, &strp, NULL);
   1844 }
   1845 
   1846 static void
   1847 gen_const_struct_at(struct gen_context *ctx,
   1848 	const struct expression *expr, struct gen_value out)
   1849 {
   1850 	// TODO: Merge me into constant expressions
   1851 	struct qbe_value base = mkqval(ctx, &out);
   1852 
   1853 	struct gen_value ftemp = mkgtemp(ctx, &builtin_type_void, "field.%d");
   1854 	for (const struct struct_constant *field = expr->constant._struct;
   1855 			field; field = field->next) {
   1856 		assert(field->value);
   1857 
   1858 		struct qbe_value offs = constl(field->field->offset);
   1859 		ftemp.type = field->value->result;
   1860 		struct qbe_value ptr = mklval(ctx, &ftemp);
   1861 		pushi(ctx->current, &ptr, Q_ADD, &base, &offs, NULL);
   1862 		gen_expr_at(ctx, field->value, ftemp);
   1863 	}
   1864 }
   1865 
   1866 static void
   1867 gen_expr_const_at(struct gen_context *ctx,
   1868 	const struct expression *expr, struct gen_value out)
   1869 {
   1870 	if (!type_is_aggregate(type_dealias(expr->result))) {
   1871 		gen_store(ctx, out, gen_expr(ctx, expr));
   1872 		return;
   1873 	}
   1874 
   1875 	switch (type_dealias(expr->result)->storage) {
   1876 	case STORAGE_ARRAY:
   1877 		gen_const_array_at(ctx, expr, out);
   1878 		break;
   1879 	case STORAGE_STRING:
   1880 		gen_const_string_at(ctx, expr, out);
   1881 		break;
   1882 	case STORAGE_STRUCT:
   1883 		gen_const_struct_at(ctx, expr, out);
   1884 		break;
   1885 	default:
   1886 		abort(); // Invariant
   1887 	}
   1888 }
   1889 
   1890 static struct gen_value
   1891 gen_expr_const(struct gen_context *ctx, const struct expression *expr)
   1892 {
   1893 	if (type_is_aggregate(type_dealias(expr->result))) {
   1894 		struct gen_value out = mkgtemp(ctx, expr->result, "object.%d");
   1895 		struct qbe_value base = mkqval(ctx, &out);
   1896 		struct qbe_value sz = constl(expr->result->size);
   1897 		enum qbe_instr alloc = alloc_for_align(expr->result->align);
   1898 		pushprei(ctx->current, &base, alloc, &sz, NULL);
   1899 		gen_expr_at(ctx, expr, out);
   1900 		return out;
   1901 	}
   1902 
   1903 	struct gen_value val = {
   1904 		.kind = GV_CONST,
   1905 		.type = expr->result,
   1906 	};
   1907 
   1908 	// Special cases
   1909 	switch (type_dealias(expr->result)->storage) {
   1910 	case STORAGE_BOOL:
   1911 		val.wval = expr->constant.bval ? 1 : 0;
   1912 		return val;
   1913 	case STORAGE_VOID:
   1914 		return val;
   1915 	case STORAGE_NULL:
   1916 		val.lval = 0;
   1917 		return val;
   1918 	default:
   1919 		// Moving right along
   1920 		break;
   1921 	}
   1922 
   1923 	const struct qbe_type *qtype = qtype_lookup(ctx, expr->result, false);
   1924 	switch (qtype->stype) {
   1925 	case Q_BYTE:
   1926 	case Q_HALF:
   1927 	case Q_WORD:
   1928 		val.wval = (uint32_t)expr->constant.uval;
   1929 		return val;
   1930 	case Q_LONG:
   1931 		val.lval = expr->constant.uval;
   1932 		return val;
   1933 	case Q_SINGLE:
   1934 		val.sval = (float)expr->constant.fval;
   1935 		return val;
   1936 	case Q_DOUBLE:
   1937 		val.dval = expr->constant.fval;
   1938 		return val;
   1939 	case Q__VOID:
   1940 		return val;
   1941 	case Q__AGGREGATE:
   1942 		assert(0); // Invariant
   1943 	}
   1944 
   1945 	abort(); // Invariant
   1946 }
   1947 
   1948 static struct gen_value
   1949 gen_expr_defer(struct gen_context *ctx, const struct expression *expr)
   1950 {
   1951 	struct gen_defer *defer = xcalloc(1, sizeof(struct gen_defer));
   1952 	defer->expr = expr->defer.deferred;
   1953 	defer->next = ctx->scope->defers;
   1954 	ctx->scope->defers = defer;
   1955 	return gv_void;
   1956 }
   1957 
   1958 static struct gen_value
   1959 gen_expr_delete(struct gen_context *ctx, const struct expression *expr)
   1960 {
   1961 	struct gen_value object, start;
   1962 	const struct expression *dexpr = expr->delete.expr;
   1963 	if (dexpr->type == EXPR_SLICE) {
   1964 		object = gen_expr(ctx, dexpr->slice.object);
   1965 		if (dexpr->slice.start) {
   1966 			start = gen_expr(ctx, dexpr->slice.start);
   1967 		} else {
   1968 			start.kind = GV_CONST;
   1969 			start.type = &builtin_type_size;
   1970 			start.lval = 0;
   1971 		}
   1972 	} else {
   1973 		assert(dexpr->type == EXPR_ACCESS
   1974 			&& dexpr->access.type == ACCESS_INDEX);
   1975 		object = gen_expr(ctx, dexpr->access.array);
   1976 		start = gen_expr(ctx, dexpr->access.index);
   1977 	}
   1978 	object = gen_autoderef(ctx, object);
   1979 	assert(type_dealias(object.type)->storage == STORAGE_SLICE);
   1980 
   1981 	struct qbe_value qobj = mkqval(ctx, &object);
   1982 	struct qbe_value qlenptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   1983 	struct qbe_value qlen = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   1984 	struct qbe_value offset = constl(builtin_type_size.size);
   1985 	enum qbe_instr load = load_for_type(ctx, &builtin_type_size);
   1986 	pushi(ctx->current, &qlenptr, Q_ADD, &qobj, &offset, NULL);
   1987 	pushi(ctx->current, &qlen, load, &qlenptr, NULL);
   1988 
   1989 	struct qbe_value qstart = mkqval(ctx, &start);
   1990 	struct qbe_value qend = {0};
   1991 	if (dexpr->type == EXPR_SLICE) {
   1992 		if (dexpr->slice.end) {
   1993 			struct gen_value end = gen_expr(ctx, dexpr->slice.end);
   1994 			qend = mkqval(ctx, &end);
   1995 		} else {
   1996 			qend = qlen;
   1997 		}
   1998 	} else {
   1999 		struct qbe_value tmp = constl(1);
   2000 		qend = mkqtmp(ctx, qstart.type, ".%d");
   2001 		pushi(ctx->current, &qend, Q_ADD, &qstart, &tmp, NULL);
   2002 	}
   2003 
   2004 	struct qbe_value start_oob = mkqtmp(ctx, &qbe_word, ".%d");
   2005 	struct qbe_value end_oob = mkqtmp(ctx, &qbe_word, ".%d");
   2006 	struct qbe_value valid = mkqtmp(ctx, &qbe_word, ".%d");
   2007 	pushi(ctx->current, &start_oob, Q_CULEL, &qstart, &qlen, NULL);
   2008 	pushi(ctx->current, &end_oob, Q_CULEL, &qend, &qlen, NULL);
   2009 	pushi(ctx->current, &valid, Q_AND, &start_oob, &end_oob, NULL);
   2010 
   2011 	struct qbe_statement linvalid, lvalid;
   2012 	struct qbe_value binvalid = mklabel(ctx, &linvalid, ".%d");
   2013 	struct qbe_value bvalid = mklabel(ctx, &lvalid, ".%d");
   2014 
   2015 	pushi(ctx->current, NULL, Q_JNZ, &valid, &bvalid, &binvalid, NULL);
   2016 	push(&ctx->current->body, &linvalid);
   2017 	gen_fixed_abort(ctx, expr->loc, ABORT_OOB);
   2018 	push(&ctx->current->body, &lvalid);
   2019 
   2020 	struct qbe_value data = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2021 	struct qbe_value startptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2022 	struct qbe_value endptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2023 	struct qbe_value mlen = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2024 	pushi(ctx->current, &data, load, &qobj, NULL);
   2025 	struct qbe_value membsz =
   2026 		constl(type_dealias(object.type)->array.members->size);
   2027 	pushi(ctx->current, &startptr, Q_MUL, &qstart, &membsz, NULL);
   2028 	pushi(ctx->current, &startptr, Q_ADD, &startptr, &data, NULL);
   2029 	pushi(ctx->current, &endptr, Q_MUL, &qend, &membsz, NULL);
   2030 	pushi(ctx->current, &endptr, Q_ADD, &endptr, &data, NULL);
   2031 	pushi(ctx->current, &qlen, Q_SUB, &qlen, &qend, NULL);
   2032 	pushi(ctx->current, &mlen, Q_MUL, &qlen, &membsz, NULL);
   2033 
   2034 	struct qbe_value rtmemmove = mkrtfunc(ctx, "rt.memmove");
   2035 	pushi(ctx->current, NULL, Q_CALL, &rtmemmove, &startptr, &endptr, &mlen,
   2036 		NULL);
   2037 
   2038 	pushi(ctx->current, &qlen, Q_ADD, &qlen, &qstart, NULL);
   2039 	enum qbe_instr store = store_for_type(ctx, &builtin_type_size);
   2040 	pushi(ctx->current, NULL, store, &qlen, &qlenptr, NULL);
   2041 
   2042 	if (!expr->delete.is_static) {
   2043 		struct qbe_value rtunensure = mkrtfunc(ctx, "rt.unensure");
   2044 		qobj = mklval(ctx, &object);
   2045 		pushi(ctx->current, NULL, Q_CALL, &rtunensure, &qobj, &membsz,
   2046 			NULL);
   2047 	}
   2048 
   2049 	return gv_void;
   2050 }
   2051 
   2052 static struct gen_value
   2053 gen_expr_for(struct gen_context *ctx, const struct expression *expr)
   2054 {
   2055 	struct qbe_statement lloop, lbody, lafter, lend;
   2056 	struct qbe_value bloop = mklabel(ctx, &lloop, "loop.%d");
   2057 	struct qbe_value bbody = mklabel(ctx, &lbody, "body.%d");
   2058 	struct qbe_value bend = mklabel(ctx, &lend, ".%d");
   2059 	struct qbe_value bafter = mklabel(ctx, &lafter, "after.%d");
   2060 
   2061 	if (expr->_for.bindings) {
   2062 		gen_expr_binding(ctx, expr->_for.bindings);
   2063 	}
   2064 
   2065 	push_scope(ctx, expr->_for.scope);
   2066 	ctx->scope->after = &bafter;
   2067 	ctx->scope->end = &bend;
   2068 
   2069 	push(&ctx->current->body, &lloop);
   2070 	struct gen_value cond = gen_expr(ctx, expr->_for.cond);
   2071 	struct qbe_value qcond = mkqval(ctx, &cond);
   2072 	pushi(ctx->current, NULL, Q_JNZ, &qcond, &bbody, &bend, NULL);
   2073 
   2074 	push(&ctx->current->body, &lbody);
   2075 	gen_expr(ctx, expr->_for.body);
   2076 
   2077 	push(&ctx->current->body, &lafter);
   2078 	if (expr->_for.afterthought) {
   2079 		gen_expr(ctx, expr->_for.afterthought);
   2080 	}
   2081 
   2082 	pop_scope(ctx, true);
   2083 
   2084 	pushi(ctx->current, NULL, Q_JMP, &bloop, NULL);
   2085 
   2086 	push(&ctx->current->body, &lend);
   2087 	return gv_void;
   2088 }
   2089 
   2090 static struct gen_value
   2091 gen_expr_free(struct gen_context *ctx, const struct expression *expr)
   2092 {
   2093 	const struct type *type = type_dealias(expr->free.expr->result);
   2094 	struct qbe_value rtfunc = mkrtfunc(ctx, "rt.free");
   2095 	struct gen_value val = gen_expr(ctx, expr->free.expr);
   2096 	struct qbe_value qval = mkqval(ctx, &val);
   2097 	if (type->storage == STORAGE_SLICE || type->storage == STORAGE_STRING) {
   2098 		struct qbe_value lval = mklval(ctx, &val);
   2099 		qval = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2100 		pushi(ctx->current, &qval, Q_LOADL, &lval, NULL);
   2101 	}
   2102 	pushi(ctx->current, NULL, Q_CALL, &rtfunc, &qval, NULL);
   2103 	return gv_void;
   2104 }
   2105 
   2106 static struct gen_value
   2107 gen_expr_if_with(struct gen_context *ctx,
   2108 	const struct expression *expr,
   2109 	struct gen_value *out)
   2110 {
   2111 	struct gen_value gvout = gv_void;
   2112 	if (!out) {
   2113 		gvout = mkgtemp(ctx, expr->result, ".%d");
   2114 	}
   2115 
   2116 	struct qbe_statement ltrue, lfalse, lend;
   2117 	struct qbe_value btrue = mklabel(ctx, &ltrue, "true.%d");
   2118 	struct qbe_value bfalse = mklabel(ctx, &lfalse, "false.%d");
   2119 	struct qbe_value bend = mklabel(ctx, &lend, ".%d");
   2120 	struct gen_value cond = gen_expr(ctx, expr->_if.cond);
   2121 	struct qbe_value qcond = mkqval(ctx, &cond);
   2122 	pushi(ctx->current, NULL, Q_JNZ, &qcond, &btrue, &bfalse, NULL);
   2123 
   2124 	push(&ctx->current->body, &ltrue);
   2125 	struct gen_value vtrue = gen_expr_with(ctx, expr->_if.true_branch, out);
   2126 	branch_copyresult(ctx, vtrue, gvout, out);
   2127 	if (!expr->_if.true_branch->terminates) {
   2128 		pushi(ctx->current, NULL, Q_JMP, &bend, NULL);
   2129 	}
   2130 
   2131 	push(&ctx->current->body, &lfalse);
   2132 	if (expr->_if.false_branch) {
   2133 		struct gen_value vfalse = gen_expr_with(ctx,
   2134 				expr->_if.false_branch, out);
   2135 		branch_copyresult(ctx, vfalse, gvout, out);
   2136 	}
   2137 
   2138 	push(&ctx->current->body, &lend);
   2139 	return gvout;
   2140 }
   2141 
   2142 static struct gen_value
   2143 gen_expr_insert(struct gen_context *ctx, const struct expression *expr)
   2144 {
   2145 	// XXX: A lot of this code is identical to the corresponding append
   2146 	// code, maybe we can/should deduplicate it
   2147 	assert(expr->append.length == NULL);
   2148 	const struct expression *objexpr = expr->append.object;
   2149 	assert(objexpr->type == EXPR_ACCESS
   2150 			&& objexpr->access.type == ACCESS_INDEX);
   2151 	const struct expression *arexpr = objexpr->access.array;
   2152 	struct gen_value slice = gen_expr(ctx, arexpr);
   2153 	slice = gen_autoderef(ctx, slice);
   2154 	struct qbe_value qslice = mkqval(ctx, &slice);
   2155 	struct gen_value index = gen_expr(ctx, objexpr->access.index);
   2156 	struct qbe_value qindex = mkqval(ctx, &index);
   2157 
   2158 	struct qbe_value lenptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2159 	struct qbe_value prevlen = mkqtmp(ctx, ctx->arch.sz, ".%d");
   2160 	struct qbe_value offs = constl(builtin_type_size.size);
   2161 	pushi(ctx->current, &lenptr, Q_ADD, &qslice, &offs, NULL);
   2162 	enum qbe_instr load = load_for_type(ctx, &builtin_type_size);
   2163 	pushi(ctx->current, &prevlen, load, &lenptr, NULL);
   2164 
   2165 	struct qbe_value appendlen = constl(1);
   2166 	const struct type *valtype = type_dealias(expr->append.value->result);
   2167 
   2168 	struct gen_value value;
   2169 	struct qbe_value qvalue;
   2170 	if (!expr->append.is_multi || valtype->storage != STORAGE_ARRAY) {
   2171 		// We use gen_expr_at for the array case to avoid a copy
   2172 		value = gen_expr(ctx, expr->append.value);
   2173 		qvalue = mkqval(ctx, &value);
   2174 	}
   2175 
   2176 	if (expr->append.is_multi) {
   2177 		if (valtype->storage == STORAGE_ARRAY) {
   2178 			assert(valtype->array.length != SIZE_UNDEFINED);
   2179 			appendlen = constl(valtype->array.length);
   2180 		} else {
   2181 			value = gen_expr(ctx, expr->append.value);
   2182 			qvalue = mkqval(ctx, &value);
   2183 			appendlen = mkqtmp(ctx, ctx->arch.sz, ".%d");
   2184 			struct qbe_value ptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2185 			offs = constl(builtin_type_size.size);
   2186 			pushi(ctx->current, &ptr, Q_ADD, &qvalue, &offs, NULL);
   2187 			pushi(ctx->current, &appendlen, load, &ptr, NULL);
   2188 		}
   2189 	}
   2190 
   2191 	struct qbe_value newlen = mkqtmp(ctx, ctx->arch.sz, ".%d");
   2192 	pushi(ctx->current, &newlen, Q_ADD, &prevlen, &appendlen, NULL);
   2193 	enum qbe_instr store = store_for_type(ctx, &builtin_type_size);
   2194 	pushi(ctx->current, NULL, store, &newlen, &lenptr, NULL);
   2195 
   2196 	struct qbe_value ptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2197 	const struct type *mtype = type_dealias(slice.type)->array.members;
   2198 	struct qbe_value membsz = constl(mtype->size);
   2199 	if (!expr->append.is_static) {
   2200 		struct qbe_value rtfunc = mkrtfunc(ctx, "rt.ensure");
   2201 		struct qbe_value lval = mklval(ctx, &slice);
   2202 		pushi(ctx->current, NULL, Q_CALL, &rtfunc, &lval, &membsz, NULL);
   2203 	} else {
   2204 		offs = constl(builtin_type_size.size * 2);
   2205 		pushi(ctx->current, &ptr, Q_ADD, &qslice, &offs, NULL);
   2206 		struct qbe_value cap = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2207 		pushi(ctx->current, &cap, load, &ptr, NULL);
   2208 
   2209 		struct qbe_statement lvalid, linvalid;
   2210 		struct qbe_value bvalid = mklabel(ctx, &lvalid, ".%d");
   2211 		struct qbe_value binvalid = mklabel(ctx, &linvalid, ".%d");
   2212 		struct qbe_value valid = mkqtmp(ctx, &qbe_word, ".%d");
   2213 		pushi(ctx->current, &valid, Q_CULEL, &newlen, &cap, NULL);
   2214 		pushi(ctx->current, NULL, Q_JNZ, &valid, &bvalid, &binvalid, NULL);
   2215 
   2216 		push(&ctx->current->body, &linvalid);
   2217 		gen_fixed_abort(ctx, expr->loc, ABORT_OOB);
   2218 		push(&ctx->current->body, &lvalid);
   2219 	}
   2220 
   2221 	struct qbe_value base = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2222 	pushi(ctx->current, &base, load, &qslice, NULL);
   2223 
   2224 	pushi(ctx->current, &ptr, Q_MUL, &qindex, &membsz, NULL);
   2225 	pushi(ctx->current, &ptr, Q_ADD, &base, &ptr, NULL);
   2226 
   2227 	struct qbe_value nbyte = mkqtmp(ctx, ctx->arch.sz, ".%d");
   2228 	struct qbe_value dest = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2229 	struct qbe_value ncopy = mkqtmp(ctx, ctx->arch.sz, ".%d");
   2230 	pushi(ctx->current, &nbyte, Q_MUL, &appendlen, &membsz, NULL);
   2231 	pushi(ctx->current, &ncopy, Q_SUB, &prevlen, &qindex, NULL);
   2232 	pushi(ctx->current, &ncopy, Q_MUL, &ncopy, &membsz, NULL);
   2233 	pushi(ctx->current, &dest, Q_ADD, &ptr, &nbyte, NULL);
   2234 
   2235 	struct qbe_value rtfunc = mkrtfunc(ctx, "rt.memmove");
   2236 	pushi(ctx->current, NULL, Q_CALL, &rtfunc, &dest, &ptr, &ncopy, NULL);
   2237 
   2238 	struct gen_value item = {
   2239 		.kind = GV_TEMP,
   2240 		.type = mtype,
   2241 		.name = ptr.name,
   2242 	};
   2243 	if (expr->append.is_multi && valtype->storage == STORAGE_ARRAY) {
   2244 		item.type = valtype;
   2245 		gen_expr_at(ctx, expr->append.value, item);
   2246 	} else if (expr->append.is_multi && valtype->storage == STORAGE_SLICE) {
   2247 		struct qbe_value qsrc = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2248 		struct qbe_value rtfunc = mkrtfunc(ctx, "rt.memmove");
   2249 		pushi(ctx->current, &qsrc, load, &qvalue, NULL);
   2250 		pushi(ctx->current, NULL, Q_CALL, &rtfunc, &ptr, &qsrc, &nbyte, NULL);
   2251 	} else {
   2252 		gen_store(ctx, item, value);
   2253 	}
   2254 
   2255 	return gv_void;
   2256 }
   2257 
   2258 enum match_compat {
   2259 	// The case type is a member of the match object type and can be used
   2260 	// directly from the match object's tagged union storage area.
   2261 	COMPAT_SUBTYPE,
   2262 	// The case type is a tagged union which is a subset of the object type,
   2263 	// and alignment-compatible. In this case we can use the match object
   2264 	// directly.
   2265 	COMPAT_ALIGNED,
   2266 	// The case type is a tagged union which is a subset of the object type,
   2267 	// but with a different alignment. In this case we must convert the
   2268 	// match object before using it.
   2269 	COMPAT_MISALIGNED,
   2270 };
   2271 
   2272 static struct qbe_value
   2273 nested_tagged_offset(const struct type *tu, const struct type *target)
   2274 {
   2275 	// This function calculates the offset of a member in a nested tagged union
   2276 	//
   2277 	// type foo = (int | void);
   2278 	// type bar = (size | foo);
   2279 	//
   2280 	// Offset of the foo field from the start of bar is 8 and offset of int
   2281 	// inside foo is 4, so the offset of the int from the start of bar is 12
   2282 	const struct type *test = tu;
   2283 	struct qbe_value offset = constl(tu->align);
   2284 	do {
   2285 		test = tagged_select_subtype(tu, target);
   2286 		if (!test) {
   2287 			break;
   2288 		}
   2289 		if (test->id != target->id && type_dealias(test)->id != target->id) {
   2290 			offset.lval += test->align;
   2291 		}
   2292 		tu = test;
   2293 	} while (test->id != target->id && type_dealias(test)->id != target->id);
   2294 	return offset;
   2295 }
   2296 
   2297 static struct gen_value
   2298 gen_nested_match_tests(struct gen_context *ctx, struct gen_value object,
   2299 	struct qbe_value bmatch, struct qbe_value bnext,
   2300 	struct qbe_value tag, const struct type *type)
   2301 {
   2302 	// This function handles the case where we're matching against a type
   2303 	// which is a member of the tagged union, or an inner tagged union.
   2304 	//
   2305 	// type foo = (int | void);
   2306 	// type bar = (size | foo);
   2307 	//
   2308 	// let x: bar = 10i;
   2309 	// match (x) {
   2310 	// case let z: size => ...
   2311 	// case let i: int => ...
   2312 	// case	void => ...
   2313 	// };
   2314 	//
   2315 	// In the first case, we can simply test the object's tag. In the second
   2316 	// case, we have to test if the selected tag is 'foo', then check the
   2317 	// tag of the foo object for int.
   2318 	struct qbe_value *subtag = &tag;
   2319 	struct qbe_value subval = mkcopy(ctx, &object, "subval.%d");
   2320 	struct gen_value match = mkgtemp(ctx, &builtin_type_bool, ".%d");
   2321 	struct qbe_value qmatch = mkqval(ctx, &match);
   2322 	struct qbe_value temp = mkqtmp(ctx, &qbe_word, ".%d");
   2323 	const struct type *subtype = object.type;
   2324 	const struct type *test = type;
   2325 	do {
   2326 		struct qbe_statement lsubtype;
   2327 		struct qbe_value bsubtype = mklabel(ctx, &lsubtype, "subtype.%d");
   2328 		test = tagged_select_subtype(subtype, type);
   2329 		if (!test) {
   2330 			break;
   2331 		}
   2332 
   2333 		struct qbe_value id = constw(test->id);
   2334 		pushi(ctx->current, &qmatch, Q_CEQW, subtag, &id, NULL);
   2335 		pushi(ctx->current, NULL, Q_JNZ, &qmatch, &bsubtype, &bnext, NULL);
   2336 		push(&ctx->current->body, &lsubtype);
   2337 
   2338 		if (test->id != type->id && type_dealias(test)-> id != type->id) {
   2339 			struct qbe_value offs = constl(subtype->align);
   2340 			pushi(ctx->current, &subval, Q_ADD, &subval, &offs, NULL);
   2341 			pushi(ctx->current, &temp, Q_LOADUW, &subval, NULL);
   2342 			subtag = &temp;
   2343 		}
   2344 
   2345 		subtype = test;
   2346 	} while (test->id != type->id && type_dealias(test)->id != type->id);
   2347 
   2348 	pushi(ctx->current, NULL, Q_JMP, &bmatch, NULL);
   2349 	return match;
   2350 }
   2351 
   2352 static struct gen_value
   2353 gen_subset_match_tests(struct gen_context *ctx,
   2354 	struct qbe_value bmatch, struct qbe_value bnext,
   2355 	struct qbe_value tag, const struct type *type)
   2356 {
   2357 	// In this case, we're testing a case which is itself a tagged union,
   2358 	// and is a subset of the match object.
   2359 	//
   2360 	// type foo = (size | int | void);
   2361 	//
   2362 	// let x: foo = 10i;
   2363 	// match (x) {
   2364 	// case let n: (size | int) => ...
   2365 	// case void => ...
   2366 	// };
   2367 	//
   2368 	// In this situation, we test the match object's tag against each type
   2369 	// ID of the case type.
   2370 	struct gen_value match = mkgtemp(ctx, &builtin_type_bool, ".%d");
   2371 	for (const struct type_tagged_union *tu = &type->tagged; tu; tu = tu->next) {
   2372 		struct qbe_statement lnexttag;
   2373 		struct qbe_value bnexttag = mklabel(ctx, &lnexttag, ".%d");
   2374 		struct qbe_value id = constl(tu->type->id);
   2375 		struct qbe_value qmatch = mkqval(ctx, &match);
   2376 		pushi(ctx->current, &qmatch, Q_CEQW, &tag, &id, NULL);
   2377 		pushi(ctx->current, NULL, Q_JNZ, &qmatch, &bmatch, &bnexttag, NULL);
   2378 		push(&ctx->current->body, &lnexttag);
   2379 	}
   2380 	pushi(ctx->current, NULL, Q_JMP, &bnext, NULL);
   2381 	return match;
   2382 }
   2383 
   2384 static struct gen_value
   2385 gen_match_with_tagged(struct gen_context *ctx,
   2386 	const struct expression *expr,
   2387 	struct gen_value *out)
   2388 {
   2389 	struct gen_value gvout = gv_void;
   2390 	if (!out) {
   2391 		gvout = mkgtemp(ctx, expr->result, ".%d");
   2392 	}
   2393 
   2394 	const struct type *objtype = expr->match.value->result;
   2395 	struct gen_value object = gen_expr(ctx, expr->match.value);
   2396 	struct qbe_value qobject = mkqval(ctx, &object);
   2397 	struct qbe_value tag = mkqtmp(ctx, ctx->arch.sz, "tag.%d");
   2398 	enum qbe_instr load = load_for_type(ctx, &builtin_type_uint);
   2399 	pushi(ctx->current, &tag, load, &qobject, NULL);
   2400 
   2401 	struct qbe_statement lout;
   2402 	struct qbe_value bout = mklabel(ctx, &lout, ".%d");
   2403 
   2404 	struct gen_value bval;
   2405 	const struct match_case *_default = NULL;
   2406 	for (const struct match_case *_case = expr->match.cases;
   2407 			_case; _case = _case->next) {
   2408 		if (!_case->type) {
   2409 			_default = _case;
   2410 			continue;
   2411 		}
   2412 
   2413 		struct qbe_statement lmatch, lnext;
   2414 		struct qbe_value bmatch = mklabel(ctx, &lmatch, "matches.%d");
   2415 		struct qbe_value bnext = mklabel(ctx, &lnext, "next.%d");
   2416 		const struct type *subtype =
   2417 			tagged_select_subtype(objtype, _case->type);
   2418 		enum match_compat compat = COMPAT_SUBTYPE;
   2419 		if (subtype) {
   2420 			gen_nested_match_tests(ctx, object,
   2421 				bmatch, bnext, tag, _case->type);
   2422 		} else {
   2423 			assert(type_dealias(_case->type)->storage == STORAGE_TAGGED);
   2424 			assert(tagged_subset_compat(objtype, _case->type));
   2425 			if (tagged_align_compat(objtype, _case->type)) {
   2426 				compat = COMPAT_ALIGNED;
   2427 			} else {
   2428 				compat = COMPAT_MISALIGNED;
   2429 			}
   2430 			const struct type *casetype = type_dealias(_case->type);
   2431 			gen_subset_match_tests(ctx, bmatch, bnext, tag, casetype);
   2432 		}
   2433 
   2434 		push(&ctx->current->body, &lmatch);
   2435 
   2436 		if (!_case->object) {
   2437 			goto next;
   2438 		}
   2439 
   2440 		struct gen_binding *gb = xcalloc(1, sizeof(struct gen_binding));
   2441 		gb->value.kind = GV_TEMP;
   2442 		gb->value.type = _case->type;
   2443 		gb->value.name = gen_name(ctx, "binding.%d");
   2444 		gb->object = _case->object;
   2445 		gb->next = ctx->bindings;
   2446 		ctx->bindings = gb;
   2447 
   2448 		struct qbe_value qv = mklval(ctx, &gb->value);
   2449 		enum qbe_instr alloc = alloc_for_align(_case->type->align);
   2450 		struct qbe_value sz = constl(_case->type->size);
   2451 		pushprei(ctx->current, &qv, alloc, &sz, NULL);
   2452 
   2453 		struct qbe_value ptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2454 		struct gen_value src = {
   2455 			.kind = GV_TEMP,
   2456 			.type = _case->type,
   2457 			.name = ptr.name,
   2458 		};
   2459 		struct gen_value load;
   2460 		struct qbe_value offset;
   2461 		switch (compat) {
   2462 		case COMPAT_SUBTYPE:
   2463 			offset = nested_tagged_offset(object.type, _case->type);
   2464 			pushi(ctx->current, &ptr, Q_ADD, &qobject, &offset, NULL);
   2465 			load = gen_load(ctx, src);
   2466 			gen_store(ctx, gb->value, load);
   2467 			break;
   2468 		case COMPAT_ALIGNED:
   2469 			pushi(ctx->current, &ptr, Q_COPY, &qobject, NULL);
   2470 			load = gen_load(ctx, src);
   2471 			gen_store(ctx, gb->value, load);
   2472 			break;
   2473 		case COMPAT_MISALIGNED:
   2474 			;
   2475 			struct expression value = {
   2476 				.type = EXPR_GEN_VALUE,
   2477 				.result = objtype,
   2478 				.user = &object,
   2479 			};
   2480 			struct expression cast = {
   2481 				.type = EXPR_CAST,
   2482 				.result = _case->type,
   2483 				.cast = {
   2484 					.kind = C_CAST,
   2485 					.secondary = _case->type,
   2486 					.value = &value,
   2487 				},
   2488 			};
   2489 			gen_expr_at(ctx, &cast, gb->value);
   2490 			break;
   2491 		}
   2492 
   2493 next:
   2494 		bval = gen_expr_with(ctx, _case->value, out);
   2495 		branch_copyresult(ctx, bval, gvout, out);
   2496 		if (!_case->value->terminates) {
   2497 			pushi(ctx->current, NULL, Q_JMP, &bout, NULL);
   2498 		}
   2499 		push(&ctx->current->body, &lnext);
   2500 	}
   2501 
   2502 	if (_default) {
   2503 		bval = gen_expr_with(ctx, _default->value, out);
   2504 		branch_copyresult(ctx, bval, gvout, out);
   2505 	} else {
   2506 		struct qbe_statement labort;
   2507 		mklabel(ctx, &labort, ".%d");
   2508 		push(&ctx->current->body, &labort);
   2509 		gen_fixed_abort(ctx, expr->loc, ABORT_UNREACHABLE);
   2510 	}
   2511 
   2512 	push(&ctx->current->body, &lout);
   2513 	return gvout;
   2514 }
   2515 
   2516 static struct gen_value
   2517 gen_match_with_nullable(struct gen_context *ctx,
   2518 	const struct expression *expr,
   2519 	struct gen_value *out)
   2520 {
   2521 	struct gen_value gvout = gv_void;
   2522 	if (!out) {
   2523 		gvout = mkgtemp(ctx, expr->result, ".%d");
   2524 	}
   2525 
   2526 	struct qbe_statement lout;
   2527 	struct qbe_value bout = mklabel(ctx, &lout, ".%d");
   2528 	struct gen_value object = gen_expr(ctx, expr->match.value);
   2529 	struct qbe_value qobject = mkqval(ctx, &object);
   2530 
   2531 	struct gen_value bval;
   2532 	const struct match_case *_default = NULL;
   2533 	for (const struct match_case *_case = expr->match.cases;
   2534 			_case; _case = _case->next) {
   2535 		if (!_case->type) {
   2536 			_default = _case;
   2537 			continue;
   2538 		}
   2539 
   2540 		struct qbe_statement lmatch, lnext;
   2541 		struct qbe_value bmatch = mklabel(ctx, &lmatch, "matches.%d");
   2542 		struct qbe_value bnext = mklabel(ctx, &lnext, "next.%d");
   2543 
   2544 		if (_case->type->storage == STORAGE_NULL) {
   2545 			pushi(ctx->current, NULL, Q_JNZ,
   2546 				&qobject, &bnext, &bmatch, NULL);
   2547 		} else {
   2548 			pushi(ctx->current, NULL, Q_JNZ,
   2549 				&qobject, &bmatch, &bnext, NULL);
   2550 		}
   2551 
   2552 		push(&ctx->current->body, &lmatch);
   2553 
   2554 		if (!_case->object) {
   2555 			goto next;
   2556 		}
   2557 
   2558 		struct gen_binding *gb = xcalloc(1, sizeof(struct gen_binding));
   2559 		gb->value = mkgtemp(ctx, _case->type, "binding.%d");
   2560 		gb->object = _case->object;
   2561 		gb->next = ctx->bindings;
   2562 		ctx->bindings = gb;
   2563 
   2564 		enum qbe_instr store = store_for_type(ctx, _case->type);
   2565 		enum qbe_instr alloc = alloc_for_align(_case->type->align);
   2566 		struct qbe_value qv = mkqval(ctx, &gb->value);
   2567 		struct qbe_value sz = constl(_case->type->size);
   2568 		pushprei(ctx->current, &qv, alloc, &sz, NULL);
   2569 		pushi(ctx->current, NULL, store, &qobject, &qv, NULL);
   2570 
   2571 next:
   2572 		bval = gen_expr_with(ctx, _case->value, out);
   2573 		branch_copyresult(ctx, bval, gvout, out);
   2574 		if (!_case->value->terminates) {
   2575 			pushi(ctx->current, NULL, Q_JMP, &bout, NULL);
   2576 		}
   2577 		push(&ctx->current->body, &lnext);
   2578 	}
   2579 
   2580 	if (_default) {
   2581 		bval = gen_expr_with(ctx, _default->value, out);
   2582 		branch_copyresult(ctx, bval, gvout, out);
   2583 		if (!_default->value->terminates) {
   2584 			pushi(ctx->current, NULL, Q_JMP, &bout, NULL);
   2585 		}
   2586 	}
   2587 
   2588 	struct qbe_statement labort;
   2589 	mklabel(ctx, &labort, ".%d");
   2590 	push(&ctx->current->body, &labort);
   2591 	gen_fixed_abort(ctx, expr->loc, ABORT_UNREACHABLE);
   2592 
   2593 	push(&ctx->current->body, &lout);
   2594 	return gvout;
   2595 }
   2596 
   2597 static struct gen_value
   2598 gen_expr_match_with(struct gen_context *ctx,
   2599 	const struct expression *expr,
   2600 	struct gen_value *out)
   2601 {
   2602 	const struct type *objtype = expr->match.value->result;
   2603 	switch (type_dealias(objtype)->storage) {
   2604 	case STORAGE_POINTER:
   2605 		return gen_match_with_nullable(ctx, expr, out);
   2606 	case STORAGE_TAGGED:
   2607 		return gen_match_with_tagged(ctx, expr, out);
   2608 	default: abort(); // Invariant
   2609 	}
   2610 }
   2611 
   2612 static struct gen_value
   2613 gen_expr_measure(struct gen_context *ctx, const struct expression *expr)
   2614 {
   2615 	size_t len;
   2616 	struct gen_value gv, temp;
   2617 	const struct type *type;
   2618 	const struct expression *value = expr->measure.value;
   2619 	switch (expr->measure.op) {
   2620 	case M_LEN:
   2621 		type = type_dealias(type_dereference(value->result));
   2622 		switch (type->storage) {
   2623 		case STORAGE_ARRAY:
   2624 			len = type->array.length;
   2625 			assert(len != SIZE_UNDEFINED);
   2626 			return (struct gen_value){
   2627 				.kind = GV_CONST,
   2628 				.type = &builtin_type_size,
   2629 				.lval = len,
   2630 			};
   2631 		case STORAGE_SLICE:
   2632 		case STORAGE_STRING:
   2633 			gv = gen_expr(ctx, value);
   2634 			gv = gen_autoderef(ctx, gv);
   2635 			temp = mkgtemp(ctx, &builtin_type_size, ".%d");
   2636 			struct qbe_value qv = mkqval(ctx, &gv),
   2637 				qtemp = mkqval(ctx, &temp),
   2638 				offs = constl(builtin_type_size.size);
   2639 			enum qbe_instr load = load_for_type(ctx,
   2640 				&builtin_type_size);
   2641 			pushi(ctx->current, &qtemp, Q_ADD, &qv, &offs, NULL);
   2642 			pushi(ctx->current, &qtemp, load, &qtemp, NULL);
   2643 			return temp;
   2644 		default:
   2645 			abort(); // Invariant
   2646 		}
   2647 		break;
   2648 	case M_SIZE:
   2649 		return (struct gen_value){
   2650 			.kind = GV_CONST,
   2651 			.type = &builtin_type_size,
   2652 			.lval = expr->measure.dimensions.size,
   2653 		};
   2654 	case M_OFFSET:
   2655 		if (expr->measure.value->access.type == ACCESS_FIELD) {
   2656 			return (struct gen_value){
   2657 				.kind = GV_CONST,
   2658 				.type = &builtin_type_size,
   2659 				.lval = expr->measure.value->access.field->offset,
   2660 			};
   2661 		} else {
   2662 			assert(expr->measure.value->access.type == ACCESS_TUPLE);
   2663 			return (struct gen_value){
   2664 				.kind = GV_CONST,
   2665 				.type = &builtin_type_size,
   2666 				.lval = expr->measure.value->access.tvalue->offset,
   2667 			};
   2668 		}
   2669 	}
   2670 	abort(); // Invariant
   2671 }
   2672 
   2673 static struct gen_value
   2674 gen_expr_return(struct gen_context *ctx, const struct expression *expr)
   2675 {
   2676 	struct gen_value ret = gen_expr(ctx, expr->_return.value);
   2677 	for (struct gen_scope *scope = ctx->scope; scope; scope = scope->parent) {
   2678 		gen_defers(ctx, scope);
   2679 	}
   2680 	if (type_dealias(ret.type)->storage == STORAGE_VOID) {
   2681 		pushi(ctx->current, NULL, Q_RET, NULL);
   2682 	} else {
   2683 		struct qbe_value qret = mkqval(ctx, &ret);
   2684 		pushi(ctx->current, NULL, Q_RET, &qret, NULL);
   2685 	}
   2686 	return gv_void;
   2687 }
   2688 
   2689 static void
   2690 gen_expr_struct_at(struct gen_context *ctx,
   2691 	const struct expression *expr,
   2692 	struct gen_value out)
   2693 {
   2694 	// TODO: Merge me into constant expressions
   2695 	struct qbe_value base = mkqval(ctx, &out);
   2696 
   2697 	if (expr->_struct.autofill) {
   2698 		struct qbe_value rtfunc = mkrtfunc(ctx, "rt.memset");
   2699 		struct qbe_value size =
   2700 			constl(expr->result->size), zero = constl(0);
   2701 		struct qbe_value base = mklval(ctx, &out);
   2702 		pushi(ctx->current, NULL, Q_CALL, &rtfunc,
   2703 			&base, &zero, &size, NULL);
   2704 	}
   2705 
   2706 	struct gen_value ftemp = mkgtemp(ctx, &builtin_type_void, "field.%d");
   2707 	for (const struct expr_struct_field *field = expr->_struct.fields;
   2708 			field; field = field->next) {
   2709 		if (!field->value) {
   2710 			assert(expr->_struct.autofill);
   2711 			continue;
   2712 		}
   2713 
   2714 		struct qbe_value offs = constl(field->field->offset);
   2715 		ftemp.type = field->value->result;
   2716 		struct qbe_value ptr = mklval(ctx, &ftemp);
   2717 		pushi(ctx->current, &ptr, Q_ADD, &base, &offs, NULL);
   2718 		gen_expr_at(ctx, field->value, ftemp);
   2719 	}
   2720 }
   2721 
   2722 static struct gen_value
   2723 gen_expr_switch_with(struct gen_context *ctx,
   2724 	const struct expression *expr,
   2725 	struct gen_value *out)
   2726 {
   2727 	struct gen_value gvout = gv_void;
   2728 	if (!out) {
   2729 		gvout = mkgtemp(ctx, expr->result, ".%d");
   2730 	}
   2731 
   2732 	struct qbe_statement lout;
   2733 	struct qbe_value bout = mklabel(ctx, &lout, ".%d");
   2734 	struct gen_value value = gen_expr(ctx, expr->_switch.value);
   2735 
   2736 	struct gen_value bval;
   2737 	const struct switch_case *_default = NULL;
   2738 	for (const struct switch_case *_case = expr->_switch.cases;
   2739 			_case; _case = _case->next) {
   2740 		if (!_case->options) {
   2741 			_default = _case;
   2742 			continue;
   2743 		}
   2744 
   2745 		struct qbe_statement lmatch, lnextcase;
   2746 		struct qbe_value bmatch = mklabel(ctx, &lmatch, "matches.%d");
   2747 		struct qbe_value bnextcase = mklabel(ctx, &lnextcase, "next.%d");
   2748 
   2749 		for (struct case_option *opt = _case->options;
   2750 				opt; opt = opt->next) {
   2751 			struct qbe_statement lnextopt;
   2752 			struct qbe_value bnextopt = mklabel(ctx, &lnextopt, ".%d");
   2753 			struct gen_value test = gen_expr_const(ctx, opt->value);
   2754 			struct expression lvalue = {
   2755 				.type = EXPR_GEN_VALUE,
   2756 				.result = value.type,
   2757 				.user = &value,
   2758 			}, rvalue = {
   2759 				.type = EXPR_GEN_VALUE,
   2760 				.result = test.type,
   2761 				.user = &test,
   2762 			}, compare = {
   2763 				.type = EXPR_BINARITHM,
   2764 				.result = &builtin_type_bool,
   2765 				.binarithm = {
   2766 					.op = BIN_LEQUAL,
   2767 					.lvalue = &lvalue,
   2768 					.rvalue = &rvalue,
   2769 				},
   2770 			};
   2771 			struct gen_value match = gen_expr(ctx, &compare);
   2772 			struct qbe_value cond = mkqval(ctx, &match);
   2773 			pushi(ctx->current, NULL, Q_JNZ,
   2774 				&cond, &bmatch, &bnextopt, NULL);
   2775 			push(&ctx->current->body, &lnextopt);
   2776 		}
   2777 
   2778 		pushi(ctx->current, NULL, Q_JMP, &bnextcase, NULL);
   2779 		push(&ctx->current->body, &lmatch);
   2780 		bval = gen_expr_with(ctx, _case->value, out);
   2781 		branch_copyresult(ctx, bval, gvout, out);
   2782 		if (!_case->value->terminates) {
   2783 			pushi(ctx->current, NULL, Q_JMP, &bout, NULL);
   2784 		}
   2785 		push(&ctx->current->body, &lnextcase);
   2786 	}
   2787 
   2788 	if (_default) {
   2789 		bval = gen_expr_with(ctx, _default->value, out);
   2790 		branch_copyresult(ctx, bval, gvout, out);
   2791 		if (!_default->value->terminates) {
   2792 			pushi(ctx->current, NULL, Q_JMP, &bout, NULL);
   2793 		}
   2794 	}
   2795 
   2796 	struct qbe_statement labort;
   2797 	mklabel(ctx, &labort, ".%d");
   2798 	push(&ctx->current->body, &labort);
   2799 	gen_fixed_abort(ctx, expr->loc, ABORT_UNREACHABLE);
   2800 
   2801 	push(&ctx->current->body, &lout);
   2802 	return gvout;
   2803 }
   2804 
   2805 static void
   2806 gen_expr_slice_at(struct gen_context *ctx,
   2807 	const struct expression *expr,
   2808 	struct gen_value out)
   2809 {
   2810 	struct gen_value object = gen_expr(ctx, expr->slice.object);
   2811 	object = gen_autoderef(ctx, object);
   2812 	const struct type *srctype = type_dealias(object.type);
   2813 
   2814 	bool check_bounds = true;
   2815 	struct gen_value length;
   2816 	struct qbe_value qlength;
   2817 	struct qbe_value qbase;
   2818 	struct qbe_value qobject = mkqval(ctx, &object);
   2819 	struct qbe_value offset = constl(builtin_type_size.size);
   2820 	struct qbe_value qptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2821 	switch (srctype->storage) {
   2822 	case STORAGE_ARRAY:
   2823 		if (srctype->array.length != SIZE_UNDEFINED) {
   2824 			length = (struct gen_value){
   2825 				.kind = GV_CONST,
   2826 				.type = &builtin_type_size,
   2827 				.lval = srctype->array.length,
   2828 			};
   2829 			qlength = mkqval(ctx, &length);
   2830 		} else {
   2831 			assert(expr->slice.end);
   2832 			check_bounds = false;
   2833 		}
   2834 		qbase = mkqval(ctx, &object);
   2835 		break;
   2836 	case STORAGE_SLICE:
   2837 		qbase = mkqtmp(ctx, ctx->arch.sz, "base.%d");
   2838 		enum qbe_instr load = load_for_type(ctx, &builtin_type_size);
   2839 		pushi(ctx->current, &qbase, load, &qobject, NULL);
   2840 		length = mkgtemp(ctx, &builtin_type_size, "len.%d");
   2841 		qlength = mkqval(ctx, &length);
   2842 		pushi(ctx->current, &qptr, Q_ADD, &qobject, &offset, NULL);
   2843 		pushi(ctx->current, &qlength, load, &qptr, NULL);
   2844 		break;
   2845 	default: abort(); // Invariant
   2846 	}
   2847 
   2848 	struct gen_value start;
   2849 	if (expr->slice.start) {
   2850 		start = gen_expr(ctx, expr->slice.start);
   2851 	} else {
   2852 		start = (struct gen_value){
   2853 			.kind = GV_CONST,
   2854 			.type = &builtin_type_size,
   2855 			.lval = 0,
   2856 		};
   2857 	}
   2858 
   2859 	struct gen_value end;
   2860 	if (expr->slice.end) {
   2861 		end = gen_expr(ctx, expr->slice.end);
   2862 	} else {
   2863 		end = length;
   2864 	}
   2865 
   2866 	struct qbe_value qstart = mkqval(ctx, &start);
   2867 	struct qbe_value qend = mkqval(ctx, &end);
   2868 
   2869 	if (check_bounds) {
   2870 		struct qbe_value start_oob = mkqtmp(ctx, &qbe_word, ".%d");
   2871 		struct qbe_value end_oob = mkqtmp(ctx, &qbe_word, ".%d");
   2872 		struct qbe_value valid = mkqtmp(ctx, &qbe_word, ".%d");
   2873 		pushi(ctx->current, &start_oob, Q_CULEL, &qstart, &qlength, NULL);
   2874 		pushi(ctx->current, &end_oob, Q_CULEL, &qend, &qlength, NULL);
   2875 		pushi(ctx->current, &valid, Q_AND, &start_oob, &end_oob, NULL);
   2876 
   2877 		struct qbe_statement linvalid, lvalid;
   2878 		struct qbe_value binvalid = mklabel(ctx, &linvalid, ".%d");
   2879 		struct qbe_value bvalid = mklabel(ctx, &lvalid, ".%d");
   2880 
   2881 		pushi(ctx->current, NULL, Q_JNZ, &valid, &bvalid, &binvalid, NULL);
   2882 		push(&ctx->current->body, &linvalid);
   2883 		gen_fixed_abort(ctx, expr->loc, ABORT_OOB);
   2884 		push(&ctx->current->body, &lvalid);
   2885 	}
   2886 
   2887 	struct qbe_value isz = constl(srctype->array.members->size);
   2888 
   2889 	struct qbe_value qout = mkqval(ctx, &out);
   2890 	struct qbe_value data = mkqtmp(ctx, ctx->arch.ptr, "data.%d");
   2891 	pushi(ctx->current, &data, Q_MUL, &qstart, &isz, NULL);
   2892 	pushi(ctx->current, &data, Q_ADD, &qbase, &data, NULL);
   2893 
   2894 	struct qbe_value newlen = mkqtmp(ctx, ctx->arch.sz, "newlen.%d");
   2895 	pushi(ctx->current, &newlen, Q_SUB, &qend, &qstart, NULL);
   2896 	struct qbe_value newcap = mkqtmp(ctx, ctx->arch.sz, "newcap.%d");
   2897 	if (check_bounds) {
   2898 		pushi(ctx->current, &newcap, Q_SUB, &qlength, &qstart, NULL);
   2899 	} else {
   2900 		pushi(ctx->current, &newcap, Q_COPY, &newlen, NULL);
   2901 	}
   2902 
   2903 	enum qbe_instr store = store_for_type(ctx, &builtin_type_size);
   2904 	pushi(ctx->current, NULL, store, &data, &qout, NULL);
   2905 	pushi(ctx->current, &qptr, Q_ADD, &qout, &offset, NULL);
   2906 	pushi(ctx->current, NULL, store, &newlen, &qptr, NULL);
   2907 	offset = constl(builtin_type_size.size * 2);
   2908 	pushi(ctx->current, &qptr, Q_ADD, &qout, &offset, NULL);
   2909 	pushi(ctx->current, NULL, store, &newcap, &qptr, NULL);
   2910 }
   2911 
   2912 static void
   2913 gen_expr_tuple_at(struct gen_context *ctx,
   2914 	const struct expression *expr,
   2915 	struct gen_value out)
   2916 {
   2917 	// TODO: Merge me into constant expressions
   2918 	struct qbe_value base = mkqval(ctx, &out);
   2919 
   2920 	const struct type *type = type_dealias(expr->result);
   2921 	struct gen_value vtemp = mkgtemp(ctx, &builtin_type_void, "value.%d");
   2922 	const struct expression_tuple *value = &expr->tuple;
   2923 	for (const struct type_tuple *tuple = &type->tuple;
   2924 			tuple; tuple = tuple->next) {
   2925 		struct qbe_value offs = constl(tuple->offset);
   2926 		vtemp.type = value->value->result;
   2927 		struct qbe_value ptr = mklval(ctx, &vtemp);
   2928 		pushi(ctx->current, &ptr, Q_ADD, &base, &offs, NULL);
   2929 		gen_expr_at(ctx, value->value, vtemp);
   2930 		value = value->next;
   2931 	}
   2932 }
   2933 
   2934 static struct gen_value
   2935 gen_expr_unarithm(struct gen_context *ctx,
   2936 	const struct expression *expr)
   2937 {
   2938 	struct gen_value val, temp;
   2939 	struct qbe_value qval, qtmp;
   2940 	const struct expression *operand = expr->unarithm.operand;
   2941 	switch (expr->unarithm.op) {
   2942 	case UN_ADDRESS:
   2943 		if (operand->type == EXPR_ACCESS) {
   2944 			val = gen_expr_access_addr(ctx, operand);
   2945 			val.type = expr->result;
   2946 			return val;
   2947 		}
   2948 		struct gen_value val = mkgtemp(ctx, operand->result, ".%d");
   2949 		struct qbe_value qv = mklval(ctx, &val);
   2950 		struct qbe_value sz = constl(val.type->size);
   2951 		enum qbe_instr alloc = alloc_for_align(val.type->align);
   2952 		pushprei(ctx->current, &qv, alloc, &sz, NULL);
   2953 		gen_expr_at(ctx, operand, val);
   2954 		val.type = expr->result;
   2955 		return val;
   2956 	case UN_DEREF:
   2957 		val = gen_expr(ctx, operand);
   2958 		assert(type_dealias(val.type)->storage == STORAGE_POINTER);
   2959 		val.type = type_dealias(val.type)->pointer.referent;
   2960 		return gen_load(ctx, val);
   2961 	case UN_BNOT:
   2962 		val = gen_expr(ctx, operand);
   2963 		temp = mkgtemp(ctx, operand->result, ".%d");
   2964 		qval = mkqval(ctx, &val), qtmp = mkqval(ctx, &temp);
   2965 		struct qbe_value ones = constl((uint64_t)-1);
   2966 		pushi(ctx->current, &qtmp, Q_XOR, &qval, &ones, NULL);
   2967 		return temp;
   2968 	case UN_LNOT:
   2969 		val = gen_expr(ctx, operand);
   2970 		temp = mkgtemp(ctx, operand->result, ".%d");
   2971 		qval = mkqval(ctx, &val), qtmp = mkqval(ctx, &temp);
   2972 		struct qbe_value zerow = constw(0);
   2973 		pushi(ctx->current, &qtmp, Q_CEQW, &qval, &zerow, NULL);
   2974 		return temp;
   2975 	case UN_MINUS:
   2976 		val = gen_expr(ctx, operand);
   2977 		temp = mkgtemp(ctx, operand->result, ".%d");
   2978 		qval = mkqval(ctx, &val), qtmp = mkqval(ctx, &temp);
   2979 		pushi(ctx->current, &qtmp, Q_NEG, &qval, NULL);
   2980 		return temp;
   2981 	case UN_PLUS:
   2982 		return gen_expr(ctx, operand);
   2983 	}
   2984 	abort(); // Invariant
   2985 }
   2986 
   2987 static struct gen_value
   2988 gen_expr_vaarg(struct gen_context *ctx,
   2989 	const struct expression *expr)
   2990 {
   2991 	// XXX: qbe only supports variadic base types, should check for this
   2992 	struct gen_value result = mkgtemp(ctx, expr->result, ".%d");
   2993 	struct qbe_value qresult = mkqval(ctx, &result);
   2994 	struct gen_value ap = gen_expr(ctx, expr->vaarg.ap);
   2995 	struct qbe_value qap = mkqval(ctx, &ap);
   2996 	pushi(ctx->current, &qresult, Q_VAARG, &qap, NULL);
   2997 	return result;
   2998 }
   2999 
   3000 static void
   3001 gen_expr_vastart_at(struct gen_context *ctx,
   3002 	const struct expression *expr,
   3003 	struct gen_value out)
   3004 {
   3005 	struct qbe_value base = mklval(ctx, &out);
   3006 	pushi(ctx->current, NULL, Q_VASTART, &base, NULL);
   3007 }
   3008 
   3009 static struct gen_value
   3010 gen_expr(struct gen_context *ctx, const struct expression *expr)
   3011 {
   3012 	switch ((int)expr->type) {
   3013 	case EXPR_ACCESS:
   3014 		return gen_expr_access(ctx, expr);
   3015 	case EXPR_ALLOC:
   3016 		return gen_expr_alloc_with(ctx, expr, NULL);
   3017 	case EXPR_APPEND:
   3018 		return gen_expr_append(ctx, expr);
   3019 	case EXPR_ASSERT:
   3020 		return gen_expr_assert(ctx, expr);
   3021 	case EXPR_ASSIGN:
   3022 		return gen_expr_assign(ctx, expr);
   3023 	case EXPR_BINARITHM:
   3024 		return gen_expr_binarithm(ctx, expr);
   3025 	case EXPR_BINDING:
   3026 		return gen_expr_binding(ctx, expr);
   3027 	case EXPR_BREAK:
   3028 	case EXPR_CONTINUE:
   3029 	case EXPR_YIELD:
   3030 		return gen_expr_control(ctx, expr);
   3031 	case EXPR_CALL:
   3032 		return gen_expr_call(ctx, expr);
   3033 	case EXPR_CAST:
   3034 		return gen_expr_cast(ctx, expr);
   3035 	case EXPR_COMPOUND:
   3036 		return gen_expr_compound_with(ctx, expr, NULL);
   3037 	case EXPR_CONSTANT:
   3038 		return gen_expr_const(ctx, expr);
   3039 	case EXPR_DEFER:
   3040 		return gen_expr_defer(ctx, expr);
   3041 	case EXPR_DELETE:
   3042 		return gen_expr_delete(ctx, expr);
   3043 	case EXPR_FOR:
   3044 		return gen_expr_for(ctx, expr);
   3045 	case EXPR_FREE:
   3046 		return gen_expr_free(ctx, expr);
   3047 	case EXPR_IF:
   3048 		return gen_expr_if_with(ctx, expr, NULL);
   3049 	case EXPR_INSERT:
   3050 		return gen_expr_insert(ctx, expr);
   3051 	case EXPR_MATCH:
   3052 		return gen_expr_match_with(ctx, expr, NULL);
   3053 	case EXPR_MEASURE:
   3054 		return gen_expr_measure(ctx, expr);
   3055 	case EXPR_PROPAGATE:
   3056 		assert(0); // Lowered in check (for now?)
   3057 	case EXPR_RETURN:
   3058 		return gen_expr_return(ctx, expr);
   3059 	case EXPR_SWITCH:
   3060 		return gen_expr_switch_with(ctx, expr, NULL);
   3061 	case EXPR_UNARITHM:
   3062 		return gen_expr_unarithm(ctx, expr);
   3063 	case EXPR_VAARG:
   3064 		return gen_expr_vaarg(ctx, expr);
   3065 	case EXPR_VAEND:
   3066 		return gv_void; // no-op
   3067 	case EXPR_SLICE:
   3068 	case EXPR_STRUCT:
   3069 	case EXPR_TUPLE:
   3070 	case EXPR_VASTART:
   3071 		break; // Prefers -at style
   3072 	// gen-specific psuedo-expressions
   3073 	case EXPR_GEN_VALUE:
   3074 		return *(struct gen_value *)expr->user;
   3075 	}
   3076 
   3077 	struct gen_value out = mkgtemp(ctx, expr->result, "object.%d");
   3078 	struct qbe_value base = mkqval(ctx, &out);
   3079 	struct qbe_value sz = constl(expr->result->size);
   3080 	enum qbe_instr alloc = alloc_for_align(expr->result->align);
   3081 	pushprei(ctx->current, &base, alloc, &sz, NULL);
   3082 	gen_expr_at(ctx, expr, out);
   3083 	return out;
   3084 }
   3085 
   3086 static void
   3087 gen_expr_at(struct gen_context *ctx,
   3088 	const struct expression *expr,
   3089 	struct gen_value out)
   3090 {
   3091 	assert(out.kind != GV_CONST);
   3092 
   3093 	switch (expr->type) {
   3094 	case EXPR_ALLOC:
   3095 		gen_expr_alloc_with(ctx, expr, &out);
   3096 		return;
   3097 	case EXPR_CAST:
   3098 		gen_expr_cast_at(ctx, expr, out);
   3099 		return;
   3100 	case EXPR_COMPOUND:
   3101 		gen_expr_compound_with(ctx, expr, &out);
   3102 		return;
   3103 	case EXPR_CONSTANT:
   3104 		gen_expr_const_at(ctx, expr, out);
   3105 		return;
   3106 	case EXPR_IF:
   3107 		gen_expr_if_with(ctx, expr, &out);
   3108 		return;
   3109 	case EXPR_MATCH:
   3110 		gen_expr_match_with(ctx, expr, &out);
   3111 		return;
   3112 	case EXPR_SLICE:
   3113 		gen_expr_slice_at(ctx, expr, out);
   3114 		return;
   3115 	case EXPR_STRUCT:
   3116 		gen_expr_struct_at(ctx, expr, out);
   3117 		return;
   3118 	case EXPR_SWITCH:
   3119 		gen_expr_switch_with(ctx, expr, &out);
   3120 		return;
   3121 	case EXPR_TUPLE:
   3122 		gen_expr_tuple_at(ctx, expr, out);
   3123 		return;
   3124 	case EXPR_VASTART:
   3125 		gen_expr_vastart_at(ctx, expr, out);
   3126 		return;
   3127 	default:
   3128 		break; // Prefers non-at style
   3129 	}
   3130 
   3131 	struct gen_value result = gen_expr(ctx, expr);
   3132 	if (!expr->terminates) {
   3133 		gen_store(ctx, out, result);
   3134 	}
   3135 }
   3136 
   3137 static struct gen_value
   3138 gen_expr_with(struct gen_context *ctx,
   3139 	const struct expression *expr,
   3140 	struct gen_value *out)
   3141 {
   3142 	if (out) {
   3143 		gen_expr_at(ctx, expr, *out);
   3144 		return *out;
   3145 	}
   3146 	return gen_expr(ctx, expr);
   3147 }
   3148 
   3149 static struct qbe_data_item *gen_data_item(struct gen_context *,
   3150 	struct expression *, struct qbe_data_item *);
   3151 
   3152 static void
   3153 gen_function_decl(struct gen_context *ctx, const struct declaration *decl)
   3154 {
   3155 	const struct function_decl *func = &decl->func;
   3156 	const struct type *fntype = func->type;
   3157 	if (func->body == NULL) {
   3158 		return; // Prototype
   3159 	}
   3160 
   3161 	struct qbe_def *qdef = xcalloc(1, sizeof(struct qbe_def));
   3162 	qdef->kind = Q_FUNC;
   3163 	qdef->exported = decl->exported;
   3164 	ctx->current = &qdef->func;
   3165 
   3166 	if (func->flags & FN_TEST) {
   3167 		qdef->name = gen_name(ctx, "testfunc.%d");
   3168 	} else if (func->flags & FN_INIT) {
   3169 		qdef->name = gen_name(ctx, "initfunc.%d");
   3170 	} else if (func->flags & FN_FINI) {
   3171 		qdef->name = gen_name(ctx, "finifunc.%d");
   3172 	} else {
   3173 		qdef->name = decl->symbol ? strdup(decl->symbol)
   3174 			: ident_to_sym(&decl->ident);
   3175 	}
   3176 
   3177 	struct qbe_statement start_label = {0};
   3178 	mklabel(ctx, &start_label, "start.%d");
   3179 	push(&qdef->func.prelude, &start_label);
   3180 
   3181 	if (type_dealias(fntype->func.result)->storage != STORAGE_VOID) {
   3182 		qdef->func.returns = qtype_lookup(
   3183 			ctx, fntype->func.result, false);
   3184 	} else {
   3185 		qdef->func.returns = &qbe_void;
   3186 	}
   3187 	if (fntype->func.variadism == VARIADISM_C) {
   3188 		qdef->func.variadic = true;
   3189 	}
   3190 
   3191 	struct qbe_func_param *param, **next = &qdef->func.params;
   3192 	for (struct scope_object *obj = decl->func.scope->objects;
   3193 			obj; obj = obj->lnext) {
   3194 		const struct type *type = obj->type;
   3195 		param = *next = xcalloc(1, sizeof(struct qbe_func_param));
   3196 		assert(!obj->ident.ns); // Invariant
   3197 		param->name = strdup(obj->ident.name);
   3198 		param->type = qtype_lookup(ctx, type, false);
   3199 
   3200 		struct gen_binding *gb =
   3201 			xcalloc(1, sizeof(struct gen_binding));
   3202 		gb->value.kind = GV_TEMP;
   3203 		gb->value.type = type;
   3204 		gb->object = obj;
   3205 		if (type_is_aggregate(type)) {
   3206 			// No need to copy to stack
   3207 			gb->value.name = strdup(param->name);
   3208 		} else {
   3209 			gb->value.name = gen_name(ctx, "param.%d");
   3210 
   3211 			struct qbe_value qv = mklval(ctx, &gb->value);
   3212 			struct qbe_value sz = constl(type->size);
   3213 			enum qbe_instr alloc = alloc_for_align(type->align);
   3214 			pushprei(ctx->current, &qv, alloc, &sz, NULL);
   3215 			struct gen_value src = {
   3216 				.kind = GV_TEMP,
   3217 				.type = type,
   3218 				.name = param->name,
   3219 			};
   3220 			gen_store(ctx, gb->value, src);
   3221 		}
   3222 
   3223 		gb->next = ctx->bindings;
   3224 		ctx->bindings = gb;
   3225 		next = &param->next;
   3226 	}
   3227 
   3228 	pushl(&qdef->func, &ctx->id, "body.%d");
   3229 	struct gen_value ret = gen_expr(ctx, decl->func.body);
   3230 
   3231 	if (decl->func.body->terminates) {
   3232 		// XXX: This is a bit hacky, to appease qbe
   3233 		size_t ln = ctx->current->body.ln;
   3234 		struct qbe_statement *last = &ctx->current->body.stmts[ln - 1];
   3235 		if (last->type != Q_INSTR || last->instr != Q_RET) {
   3236 			pushi(ctx->current, NULL, Q_RET, NULL);
   3237 		}
   3238 	} else if (type_dealias(fntype->func.result)->storage != STORAGE_VOID) {
   3239 		struct qbe_value qret = mkqval(ctx, &ret);
   3240 		pushi(ctx->current, NULL, Q_RET, &qret, NULL);
   3241 	} else {
   3242 		pushi(ctx->current, NULL, Q_RET, NULL);
   3243 	}
   3244 
   3245 	qbe_append_def(ctx->out, qdef);
   3246 
   3247 	if (func->flags & FN_INIT) {
   3248 		struct qbe_def *init = xcalloc(1, sizeof *init);
   3249 		init->kind = Q_DATA;
   3250 		init->exported = false;
   3251 		init->data.align = 8;
   3252 		init->data.section = ".init_array";
   3253 		init->data.secflags = NULL;
   3254 
   3255 		size_t n = snprintf(NULL, 0, ".init.%s", qdef->name);
   3256 		init->name = xcalloc(n + 1, 1);
   3257 		snprintf(init->name, n + 1, ".init.%s", qdef->name);
   3258 
   3259 		struct qbe_data_item dataitem = {
   3260 			.type = QD_VALUE,
   3261 			.value = {
   3262 				.kind = QV_GLOBAL,
   3263 				.type = &qbe_long,
   3264 				.name = strdup(qdef->name),
   3265 			},
   3266 			.next = NULL,
   3267 		};
   3268 		init->data.items = dataitem;
   3269 
   3270 		qbe_append_def(ctx->out, init);
   3271 	}
   3272 
   3273 	if (func->flags & FN_FINI) {
   3274 		struct qbe_def *fini = xcalloc(1, sizeof *fini);
   3275 		fini->kind = Q_DATA;
   3276 		fini->exported = false;
   3277 		fini->data.align = 8;
   3278 		fini->data.section = ".fini_array";
   3279 		fini->data.secflags = NULL;
   3280 
   3281 		size_t n = snprintf(NULL, 0, ".fini.%s", qdef->name);
   3282 		fini->name = xcalloc(n + 1, 1);
   3283 		snprintf(fini->name, n + 1, ".fini.%s", qdef->name);
   3284 
   3285 		struct qbe_data_item dataitem = {
   3286 			.type = QD_VALUE,
   3287 			.value = {
   3288 				.kind = QV_GLOBAL,
   3289 				.type = &qbe_long,
   3290 				.name = strdup(qdef->name),
   3291 			},
   3292 			.next = NULL,
   3293 		};
   3294 		fini->data.items = dataitem;
   3295 
   3296 		qbe_append_def(ctx->out, fini);
   3297 	}
   3298 
   3299 	if (func->flags & FN_TEST) {
   3300 		struct qbe_def *test = xcalloc(1, sizeof *test);
   3301 		test->kind = Q_DATA;
   3302 		test->exported = false;
   3303 		test->data.align = 8;
   3304 		test->data.section = ".test_array";
   3305 		test->data.secflags = "aw";
   3306 
   3307 		size_t n = snprintf(NULL, 0, ".test.%s", qdef->name);
   3308 		test->name = xcalloc(n + 1, 1);
   3309 		snprintf(test->name, n + 1, ".test.%s", qdef->name);
   3310 
   3311 		char *ident = identifier_unparse(&decl->ident);
   3312 
   3313 		struct qbe_data_item *dataitem = &test->data.items;
   3314 		struct expression expr = {
   3315 			.type = EXPR_CONSTANT,
   3316 			.result = &builtin_type_str,
   3317 			.constant = {
   3318 				.object = NULL,
   3319 				.string = {
   3320 					.value = ident,
   3321 					.len = strlen(ident),
   3322 				},
   3323 			},
   3324 		};
   3325 		dataitem = gen_data_item(ctx, &expr, dataitem);
   3326 
   3327 		struct qbe_data_item *next = xcalloc(1, sizeof *next);
   3328 		next->type = QD_VALUE;
   3329 		next->value.kind = QV_GLOBAL;
   3330 		next->value.type = &qbe_long;
   3331 		next->value.name = strdup(qdef->name);
   3332 		next->next = NULL;
   3333 		dataitem->next = next;
   3334 
   3335 		qbe_append_def(ctx->out, test);
   3336 	}
   3337 
   3338 	ctx->current = NULL;
   3339 }
   3340 
   3341 static struct qbe_data_item *
   3342 gen_data_item(struct gen_context *ctx, struct expression *expr,
   3343 	struct qbe_data_item *item)
   3344 {
   3345 	assert(expr->type == EXPR_CONSTANT);
   3346 
   3347 	struct qbe_def *def;
   3348 	const struct expression_constant *constant = &expr->constant;
   3349 	const struct type *type = type_dealias(expr->result);
   3350 	type = lower_const(type, NULL);
   3351 	if (constant->object) {
   3352 		item->type = QD_SYMOFFS;
   3353 		item->sym = ident_to_sym(&constant->object->ident);
   3354 		item->offset = constant->ival;
   3355 		return item;
   3356 	}
   3357 
   3358 	switch (type->storage) {
   3359 	case STORAGE_I8:
   3360 	case STORAGE_U8:
   3361 	case STORAGE_CHAR:
   3362 		item->type = QD_VALUE;
   3363 		item->value = constw((uint8_t)constant->uval);
   3364 		item->value.type = &qbe_byte;
   3365 		break;
   3366 	case STORAGE_I16:
   3367 	case STORAGE_U16:
   3368 		item->type = QD_VALUE;
   3369 		item->value = constw((uint16_t)constant->uval);
   3370 		item->value.type = &qbe_half;
   3371 		break;
   3372 	case STORAGE_BOOL:
   3373 		item->type = QD_VALUE;
   3374 		item->value = constw(constant->bval ? 1 : 0);
   3375 		break;
   3376 	case STORAGE_I32:
   3377 	case STORAGE_U32:
   3378 	case STORAGE_INT:
   3379 	case STORAGE_UINT:
   3380 	case STORAGE_RUNE:
   3381 		item->type = QD_VALUE;
   3382 		item->value = constw((uint32_t)constant->uval);
   3383 		break;
   3384 	case STORAGE_U64:
   3385 	case STORAGE_I64:
   3386 	case STORAGE_SIZE:
   3387 		item->type = QD_VALUE;
   3388 		item->value = constl((uint64_t)constant->uval);
   3389 		break;
   3390 	case STORAGE_F32:
   3391 		item->type = QD_VALUE;
   3392 		item->value = consts((float)constant->fval);
   3393 		break;
   3394 	case STORAGE_F64:
   3395 		item->type = QD_VALUE;
   3396 		item->value = constd((double)constant->fval);
   3397 		break;
   3398 	case STORAGE_UINTPTR:
   3399 	case STORAGE_POINTER:
   3400 		assert(expr->type == EXPR_CONSTANT); // TODO?
   3401 		item->type = QD_VALUE;
   3402 		switch (ctx->arch.ptr->stype) {
   3403 		case Q_LONG:
   3404 			item->value = constl((uint64_t)constant->uval);
   3405 			break;
   3406 		default: assert(0);
   3407 		}
   3408 		break;
   3409 	case STORAGE_ARRAY:
   3410 		assert(type->array.length != SIZE_UNDEFINED);
   3411 		size_t n = type->array.length;
   3412 		for (struct array_constant *c = constant->array;
   3413 				c && n; c = c->next ? c->next : c, --n) {
   3414 			item = gen_data_item(ctx, c->value, item);
   3415 			if (n > 1 || c->next) {
   3416 				item->next = xcalloc(1,
   3417 					sizeof(struct qbe_data_item));
   3418 				item = item->next;
   3419 			}
   3420 		}
   3421 		break;
   3422 	case STORAGE_STRING:
   3423 		def = xcalloc(1, sizeof(struct qbe_def));
   3424 		def->name = gen_name(ctx, "strdata.%d");
   3425 		def->kind = Q_DATA;
   3426 		def->data.align = ALIGN_UNDEFINED;
   3427 		def->data.items.type = QD_STRING;
   3428 		def->data.items.str = xcalloc(1, expr->constant.string.len);
   3429 		def->data.items.sz = expr->constant.string.len;
   3430 		memcpy(def->data.items.str, expr->constant.string.value,
   3431 			expr->constant.string.len);
   3432 
   3433 		item->type = QD_VALUE;
   3434 		if (expr->constant.string.len != 0) {
   3435 			qbe_append_def(ctx->out, def);
   3436 			item->value.kind = QV_GLOBAL;
   3437 			item->value.type = &qbe_long;
   3438 			item->value.name = strdup(def->name);
   3439 		} else {
   3440 			free(def);
   3441 			item->value = constl(0);
   3442 		}
   3443 
   3444 		item->next = xcalloc(1, sizeof(struct qbe_data_item));
   3445 		item = item->next;
   3446 		item->type = QD_VALUE;
   3447 		item->value = constl(expr->constant.string.len);
   3448 		item->next = xcalloc(1, sizeof(struct qbe_data_item));
   3449 		item = item->next;
   3450 		item->type = QD_VALUE;
   3451 		item->value = constl(expr->constant.string.len);
   3452 		break;
   3453 	case STORAGE_SLICE:
   3454 		def = xcalloc(1, sizeof(struct qbe_def));
   3455 		def->name = gen_name(ctx, "sldata.%d");
   3456 		def->kind = Q_DATA;
   3457 		def->data.align = ALIGN_UNDEFINED;
   3458 
   3459 		size_t len = 0;
   3460 		struct qbe_data_item *subitem = &def->data.items;
   3461 		for (struct array_constant *c = constant->array;
   3462 				c; c = c->next) {
   3463 			subitem = gen_data_item(ctx, c->value, subitem);
   3464 			if (c->next) {
   3465 				subitem->next = xcalloc(1,
   3466 					sizeof(struct qbe_data_item));
   3467 				subitem = subitem->next;
   3468 			}
   3469 			++len;
   3470 		}
   3471 
   3472 		item->type = QD_VALUE;
   3473 		if (len != 0) {
   3474 			qbe_append_def(ctx->out, def);
   3475 			item->value.kind = QV_GLOBAL;
   3476 			item->value.type = &qbe_long;
   3477 			item->value.name = strdup(def->name);
   3478 		} else {
   3479 			free(def);
   3480 			item->value = constl(0);
   3481 		}
   3482 
   3483 		item->next = xcalloc(1, sizeof(struct qbe_data_item));
   3484 		item = item->next;
   3485 		item->type = QD_VALUE;
   3486 		item->value = constl(len);
   3487 		item->next = xcalloc(1, sizeof(struct qbe_data_item));
   3488 		item = item->next;
   3489 		item->type = QD_VALUE;
   3490 		item->value = constl(len);
   3491 		break;
   3492 	case STORAGE_STRUCT:
   3493 		for (struct struct_constant *f = constant->_struct;
   3494 				f; f = f->next) {
   3495 			item = gen_data_item(ctx, f->value, item);
   3496 			if (f->next) {
   3497 				const struct struct_field *f1 = f->field;
   3498 				const struct struct_field *f2 = f->next->field;
   3499 				if (f2->offset != f1->offset + f1->type->size) {
   3500 					item->next = xcalloc(1,
   3501 						sizeof(struct qbe_data_item));
   3502 					item = item->next;
   3503 					item->type = QD_ZEROED;
   3504 					item->zeroed = f2->offset -
   3505 						(f1->offset + f1->type->size);
   3506 				}
   3507 
   3508 				item->next = xcalloc(1,
   3509 					sizeof(struct qbe_data_item));
   3510 				item = item->next;
   3511 			} else {
   3512 				const struct struct_field *fi = f->field;
   3513 				if (fi->offset + fi->type->size
   3514 						!= expr->result->size) {
   3515 					item->next = xcalloc(1,
   3516 						sizeof(struct qbe_data_item));
   3517 					item = item->next;
   3518 					item->type = QD_ZEROED;
   3519 					item->zeroed = expr->result->size
   3520 						- (fi->offset + fi->type->size);
   3521 				}
   3522 			}
   3523 		}
   3524 		break;
   3525 	case STORAGE_ENUM:
   3526 		switch (type->alias.type->storage) {
   3527 		case STORAGE_I8:
   3528 		case STORAGE_U8:
   3529 			item->type = QD_VALUE;
   3530 			item->value = constw((uint8_t)constant->uval);
   3531 			item->value.type = &qbe_byte;
   3532 			break;
   3533 		case STORAGE_I16:
   3534 		case STORAGE_U16:
   3535 			item->type = QD_VALUE;
   3536 			item->value = constw((uint16_t)constant->uval);
   3537 			item->value.type = &qbe_half;
   3538 			break;
   3539 		case STORAGE_I32:
   3540 		case STORAGE_U32:
   3541 		case STORAGE_INT: // XXX: arch
   3542 		case STORAGE_UINT:
   3543 			item->type = QD_VALUE;
   3544 			item->value = constw((uint32_t)constant->uval);
   3545 			break;
   3546 		case STORAGE_U64:
   3547 		case STORAGE_I64:
   3548 		case STORAGE_SIZE: // XXX: arch
   3549 		case STORAGE_UINTPTR:
   3550 			item->type = QD_VALUE;
   3551 			item->value = constl((uint64_t)constant->uval);
   3552 			break;
   3553 		default:
   3554 			assert(0);
   3555 		}
   3556 		break;
   3557 	case STORAGE_TUPLE:
   3558 		for (const struct tuple_constant *tuple = constant->tuple;
   3559 				tuple; tuple = tuple->next) {
   3560 			item = gen_data_item(ctx, tuple->value, item);
   3561 			if (tuple->next) {
   3562 				const struct type_tuple *f1 = tuple->field;
   3563 				const struct type_tuple *f2 = tuple->next->field;
   3564 				if (f2->offset != f1->offset + f1->type->size) {
   3565 					item->next = xcalloc(1,
   3566 						sizeof(struct qbe_data_item));
   3567 					item = item->next;
   3568 					item->type = QD_ZEROED;
   3569 					item->zeroed = f2->offset -
   3570 						(f1->offset + f1->type->size);
   3571 				}
   3572 
   3573 
   3574 				item->next = xcalloc(1,
   3575 					sizeof(struct qbe_data_item));
   3576 				item = item->next;
   3577 			} else {
   3578 				const struct type_tuple *fi = tuple->field;
   3579 				if (fi->offset + fi->type->size
   3580 						!= expr->result->size) {
   3581 					item->next = xcalloc(1,
   3582 						sizeof(struct qbe_data_item));
   3583 					item = item->next;
   3584 					item->type = QD_ZEROED;
   3585 					item->zeroed = expr->result->size
   3586 						- (fi->offset + fi->type->size);
   3587 				}
   3588 			}
   3589 		}
   3590 		break;
   3591 	case STORAGE_TAGGED:
   3592 		item->type = QD_VALUE;
   3593 		item->value = constw((uint32_t)constant->tagged.tag->id);
   3594 		if (type->align != builtin_type_uint.align) {
   3595 			item->next = xcalloc(1, sizeof(struct qbe_data_item));
   3596 			item = item->next;
   3597 			item->type = QD_ZEROED;
   3598 			item->zeroed = type->align - builtin_type_uint.align;
   3599 		}
   3600 		if (constant->tagged.tag->size != 0) {
   3601 			item->next = xcalloc(1, sizeof(struct qbe_data_item));
   3602 			item = item->next;
   3603 			item = gen_data_item(ctx, constant->tagged.value, item);
   3604 		}
   3605 		if (constant->tagged.tag->size < type->size - type->align) {
   3606 			item->next = xcalloc(1, sizeof(struct qbe_data_item));
   3607 			item = item->next;
   3608 			item->type = QD_ZEROED;
   3609 			item->zeroed = type->size - type->align - constant->tagged.tag->size;
   3610 		}
   3611 		break;
   3612 	case STORAGE_UNION:
   3613 	case STORAGE_ALIAS:
   3614 	case STORAGE_FCONST:
   3615 	case STORAGE_FUNCTION:
   3616 	case STORAGE_ICONST:
   3617 	case STORAGE_RCONST:
   3618 	case STORAGE_NULL:
   3619 	case STORAGE_VALIST:
   3620 	case STORAGE_VOID:
   3621 		assert(0); // Invariant
   3622 	}
   3623 
   3624 	assert(item->type != QD_VALUE || item->value.type);
   3625 	return item;
   3626 }
   3627 
   3628 static void
   3629 gen_global_decl(struct gen_context *ctx, const struct declaration *decl)
   3630 {
   3631 	assert(decl->type == DECL_GLOBAL);
   3632 	const struct global_decl *global = &decl->global;
   3633 	if (!global->value) {
   3634 		return; // Forward declaration
   3635 	}
   3636 	struct qbe_def *qdef = xcalloc(1, sizeof(struct qbe_def));
   3637 	qdef->kind = Q_DATA;
   3638 	qdef->data.align = ALIGN_UNDEFINED;
   3639 	qdef->exported = decl->exported;
   3640 	qdef->name = ident_to_sym(&decl->ident);
   3641 	gen_data_item(ctx, global->value, &qdef->data.items);
   3642 	qbe_append_def(ctx->out, qdef);
   3643 }
   3644 
   3645 static void
   3646 gen_decl(struct gen_context *ctx, const struct declaration *decl)
   3647 {
   3648 	switch (decl->type) {
   3649 	case DECL_FUNC:
   3650 		gen_function_decl(ctx, decl);
   3651 		break;
   3652 	case DECL_GLOBAL:
   3653 		gen_global_decl(ctx, decl);
   3654 		break;
   3655 	case DECL_TYPE:
   3656 	case DECL_CONST:
   3657 		break; // no-op
   3658 	}
   3659 }
   3660 
   3661 void
   3662 gen(const struct unit *unit, struct type_store *store, struct qbe_program *out)
   3663 {
   3664 	struct gen_context ctx = {
   3665 		.out = out,
   3666 		.store = store,
   3667 		.ns = unit->ns,
   3668 		.arch = {
   3669 			.ptr = &qbe_long,
   3670 			.sz = &qbe_long,
   3671 		},
   3672 	};
   3673 	ctx.out->next = &ctx.out->defs;
   3674 	const struct declarations *decls = unit->declarations;
   3675 	while (decls) {
   3676 		gen_decl(&ctx, decls->decl);
   3677 		decls = decls->next;
   3678 	}
   3679 }